Initial commit: Tagesbyte prototype
Flutter app that collects daily tech news from German and English RSS/Atom feeds, with a reader view, AI article summaries and daily briefings (Claude API or n8n webhook), paywall/sponsored filtering and ad/consent blocking. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
f12fb1dabb
159 files changed
+8738
No files matched your search
@@ -0,0 +1,250 @@
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:html/dom.dart' as dom;
|
||||
import 'package:html/parser.dart' as html_parser;
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
import '../models/article.dart';
|
||||
import 'consent_service.dart';
|
||||
import 'content_blocking.dart';
|
||||
import 'feed_service.dart';
|
||||
|
||||
enum BlockType { heading, paragraph, listItem, quote, code, image, caption }
|
||||
|
||||
/// One piece of an article in reader view. For images [text] is the URL.
|
||||
class ReaderBlock {
|
||||
const ReaderBlock(this.type, this.text);
|
||||
|
||||
final BlockType type;
|
||||
final String text;
|
||||
|
||||
bool get isText => type != BlockType.image && type != BlockType.caption;
|
||||
}
|
||||
|
||||
class ArticleContent {
|
||||
ArticleContent(this.blocks, {this.consentWall = false})
|
||||
: plainText = blocks.where((b) => b.isText).map((b) => b.text).join('\n\n');
|
||||
|
||||
static final empty = ArticleContent(const []);
|
||||
|
||||
final List<ReaderBlock> blocks;
|
||||
|
||||
/// Text handed to the AI summarizer.
|
||||
final String plainText;
|
||||
|
||||
/// The site redirected to a consent page (accept tracking or subscribe)
|
||||
/// instead of serving the article.
|
||||
final bool consentWall;
|
||||
}
|
||||
|
||||
/// Turns an article page into reader-view blocks so it can be read inside the
|
||||
/// app, and gives the AI more to work with than the feed teaser.
|
||||
class ArticleExtractor {
|
||||
ArticleExtractor({http.Client? client, ConsentService? consent})
|
||||
: _client = client ?? http.Client(),
|
||||
_consent = consent;
|
||||
|
||||
final http.Client _client;
|
||||
final ConsentService? _consent;
|
||||
|
||||
/// Hard cap on what we send to the model per article (~10k tokens).
|
||||
static const maxChars = 40000;
|
||||
|
||||
/// Loads the article page and returns whichever is longer: the extracted
|
||||
/// page or the full content some feeds ship. Never throws; on failure
|
||||
/// (paywall, bot protection, CORS on web) it falls back to feed content.
|
||||
///
|
||||
/// With [acceptConsentWalls], a redirect to a consent page the
|
||||
/// [ConsentService] knows is accepted once and the article fetched again.
|
||||
Future<ArticleContent> load(
|
||||
Article article, {
|
||||
String corsProxy = '',
|
||||
bool acceptConsentWalls = false,
|
||||
}) async {
|
||||
final base = Uri.parse(article.link);
|
||||
final fromFeed = article.contentHtml.isEmpty
|
||||
? ArticleContent.empty
|
||||
: parse(article.contentHtml, base, isFragment: true);
|
||||
|
||||
if (kIsWeb && corsProxy.isEmpty) return fromFeed;
|
||||
try {
|
||||
final url = Uri.parse(
|
||||
kIsWeb ? '$corsProxy${Uri.encodeComponent(article.link)}' : article.link);
|
||||
var page = await _fetch(url);
|
||||
final consent = _consent;
|
||||
if (page.wallUrl != null &&
|
||||
acceptConsentWalls &&
|
||||
consent != null &&
|
||||
consent.canAccept(page.wallUrl!) &&
|
||||
await consent.accept(page.wallUrl!, wallHtml: FeedService.decodeBody(page.bytes))) {
|
||||
page = await _fetch(url);
|
||||
}
|
||||
if (page.wallUrl != null) return ArticleContent(fromFeed.blocks, consentWall: true);
|
||||
if (page.status != 200) return fromFeed;
|
||||
final fromPage = parse(FeedService.decodeBody(page.bytes), base);
|
||||
return fromPage.plainText.length > fromFeed.plainText.length ? fromPage : fromFeed;
|
||||
} catch (_) {
|
||||
return fromFeed;
|
||||
}
|
||||
}
|
||||
|
||||
/// [wallUrl] is set when the request ended on a consent page.
|
||||
Future<({int status, List<int> bytes, Uri? wallUrl})> _fetch(Uri url) async {
|
||||
final request = http.Request('GET', url);
|
||||
if (!kIsWeb) {
|
||||
request.headers['User-Agent'] = 'Mozilla/5.0 (DailyTechNews)';
|
||||
final cookie = _consent?.cookieHeader(url);
|
||||
if (cookie != null) request.headers['Cookie'] = cookie;
|
||||
}
|
||||
// send() (unlike get()) keeps the final URL after redirects.
|
||||
final response = await _client.send(request).timeout(const Duration(seconds: 20));
|
||||
final finalUrl = response is http.BaseResponseWithUrl
|
||||
? (response as http.BaseResponseWithUrl).url
|
||||
: null;
|
||||
final onWall =
|
||||
finalUrl != null && ContentBlocking.consentWall.hasMatch(finalUrl.toString());
|
||||
if (response.statusCode != 200) {
|
||||
await response.stream.listen(null).cancel();
|
||||
return (status: response.statusCode, bytes: const <int>[], wallUrl: null);
|
||||
}
|
||||
final bytes = await response.stream.toBytes().timeout(const Duration(seconds: 20));
|
||||
return (status: 200, bytes: bytes, wallUrl: onWall ? finalUrl : null);
|
||||
}
|
||||
|
||||
static const _junkTags =
|
||||
'script, style, nav, header, footer, aside, form, noscript, button, svg, iframe, '
|
||||
'template, dialog, '
|
||||
// Screen-reader-only text such as "(öffnet im neuen Fenster)".
|
||||
'.sr-only, .visually-hidden, .visuallyhidden, .screen-reader-text, .go-vh';
|
||||
|
||||
/// Class/id fragments of page furniture that sits inside article bodies.
|
||||
static final _junkMarker = RegExp(
|
||||
r'comment|related|newsletter|share|social|promo|advert|paywall|subscribe|'
|
||||
r'recommend|author-box|byline|breadcrumb|tags|toolbar|consent|cookie|card|native-ad',
|
||||
caseSensitive: false,
|
||||
);
|
||||
|
||||
static const _containerTags = {'div', 'section', 'ul', 'ol', 'span', 'aside'};
|
||||
|
||||
static ArticleContent parse(String html, Uri base, {bool isFragment = false}) {
|
||||
final root = isFragment
|
||||
? (html_parser.parse('<body>$html</body>').body!)
|
||||
: html_parser.parse(html).documentElement!;
|
||||
root.querySelectorAll(_junkTags).forEach((dom.Element e) => e.remove());
|
||||
for (final e in root.querySelectorAll('*').toList()) {
|
||||
if (!_containerTags.contains(e.localName)) continue;
|
||||
final marker = '${e.className} ${e.id}';
|
||||
if (_junkMarker.hasMatch(marker)) e.remove();
|
||||
}
|
||||
|
||||
// Many sites wrap every teaser in its own <article>, so pick whichever
|
||||
// candidate container holds the most paragraph text.
|
||||
dom.Element? best;
|
||||
var bestScore = 0;
|
||||
for (final candidate
|
||||
in root.querySelectorAll('article, main, [itemprop="articleBody"]')) {
|
||||
final score = _score(candidate);
|
||||
if (score > bestScore) {
|
||||
best = candidate;
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
// The body includes navigation and comments; only use it as a last resort.
|
||||
best ??= isFragment ? root : root.querySelector('body');
|
||||
if (best == null) return ArticleContent.empty;
|
||||
|
||||
final blocks = <ReaderBlock>[];
|
||||
final seenImages = <String>{};
|
||||
var chars = 0;
|
||||
void add(BlockType type, String text) {
|
||||
if (chars > maxChars) return;
|
||||
if (type == BlockType.image) {
|
||||
if (!seenImages.add(text)) return;
|
||||
} else {
|
||||
chars += text.length;
|
||||
}
|
||||
blocks.add(ReaderBlock(type, text));
|
||||
}
|
||||
|
||||
void walk(dom.Element el) {
|
||||
switch (el.localName) {
|
||||
case 'h1':
|
||||
return; // The title is shown from the feed.
|
||||
case 'h2' || 'h3' || 'h4':
|
||||
final t = _clean(el.text);
|
||||
if (t.isNotEmpty) add(BlockType.heading, t);
|
||||
return;
|
||||
case 'p':
|
||||
final t = _clean(el.text);
|
||||
if (t.length >= 25) add(BlockType.paragraph, t);
|
||||
el.querySelectorAll('img').forEach((img) => _addImage(img, base, add));
|
||||
return;
|
||||
case 'li':
|
||||
// Nested lists: emit the inner items individually.
|
||||
if (el.querySelector('ul, ol') != null) break;
|
||||
final t = _clean(el.text);
|
||||
// Items that are nothing but a link are tables of contents or nav.
|
||||
final linkText = _clean(el.querySelectorAll('a').map((a) => a.text).join(' '));
|
||||
if (linkText == t) return;
|
||||
if (t.length >= 20) add(BlockType.listItem, t);
|
||||
return;
|
||||
case 'blockquote':
|
||||
final t = _clean(el.text);
|
||||
if (t.isNotEmpty) add(BlockType.quote, t);
|
||||
return;
|
||||
case 'pre':
|
||||
final t = el.text.trimRight();
|
||||
if (t.isNotEmpty) add(BlockType.code, t);
|
||||
return;
|
||||
case 'figcaption':
|
||||
final t = _clean(el.text);
|
||||
if (t.isNotEmpty) add(BlockType.caption, t);
|
||||
return;
|
||||
case 'img':
|
||||
_addImage(el, base, add);
|
||||
return;
|
||||
}
|
||||
for (final child in el.children) {
|
||||
walk(child);
|
||||
}
|
||||
}
|
||||
|
||||
walk(best);
|
||||
return ArticleContent(blocks);
|
||||
}
|
||||
|
||||
static int _score(dom.Element root) => root
|
||||
.querySelectorAll('p')
|
||||
.map((e) => _clean(e.text).length)
|
||||
.where((l) => l > 40)
|
||||
.fold(0, (a, b) => a + b);
|
||||
|
||||
static String _clean(String s) => s.replaceAll(RegExp(r'\s+'), ' ').trim();
|
||||
|
||||
static void _addImage(
|
||||
dom.Element img, Uri base, void Function(BlockType, String) add) {
|
||||
// Lazy-loading sites keep the real URL in data attributes.
|
||||
final raw = img.attributes['data-src'] ??
|
||||
img.attributes['data-lazy-src'] ??
|
||||
img.attributes['src'] ??
|
||||
img.attributes['srcset']?.split(',').first.trim().split(' ').first;
|
||||
if (raw == null || raw.isEmpty || raw.startsWith('data:') || raw.endsWith('.svg')) {
|
||||
return;
|
||||
}
|
||||
final width = int.tryParse(img.attributes['width'] ?? '');
|
||||
if (width != null && width < 120) return; // icons, avatars, tracking pixels
|
||||
// Images linking to another page are author avatars or related-story
|
||||
// thumbnails; links to the full-size image file are fine.
|
||||
for (var p = img.parent; p != null; p = p.parent) {
|
||||
if (p.localName == 'a') {
|
||||
final href = p.attributes['href'] ?? '';
|
||||
if (!RegExp(r'\.(jpe?g|png|webp|gif|avif)(\?|$)', caseSensitive: false)
|
||||
.hasMatch(href)) {
|
||||
return;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
final url = base.resolve(raw).toString();
|
||||
if (url.startsWith('http')) add(BlockType.image, url);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
import '../models/article.dart';
|
||||
import '../models/briefing.dart';
|
||||
import '../models/news_source.dart';
|
||||
import 'summarizer.dart';
|
||||
|
||||
/// Calls the Claude Messages API directly over HTTP (there is no official
|
||||
/// Dart SDK).
|
||||
class ClaudeSummarizer implements Summarizer {
|
||||
ClaudeSummarizer({required this.apiKey, required this.model, http.Client? client})
|
||||
: _client = client ?? http.Client();
|
||||
|
||||
final String apiKey;
|
||||
final String model;
|
||||
final http.Client _client;
|
||||
|
||||
static final _endpoint = Uri.parse('https://api.anthropic.com/v1/messages');
|
||||
|
||||
/// Models that support the server-side refusal fallback (`fallbacks: "default"`).
|
||||
static const _fallbackModels = {'claude-opus-5-5', 'claude-sonnet-5-5'};
|
||||
|
||||
static const _retryStatus = {429, 500, 502, 503, 529};
|
||||
|
||||
@override
|
||||
String get name => 'Claude ($model)';
|
||||
|
||||
@override
|
||||
Future<String> summarizeArticle(
|
||||
Article article, String text, SourceLanguage language) async {
|
||||
final system = 'You summarize tech news articles for a busy reader. '
|
||||
'Write in ${languageName(language)}, regardless of the article language. '
|
||||
'Start with a single-sentence TL;DR line prefixed with "TL;DR:", then 3-5 '
|
||||
'short bullet points starting with "• " covering the key facts, numbers and '
|
||||
'why it matters. Plain text only, no markdown headings or bold. If the text '
|
||||
'is only a teaser, summarize what is there and say that the full article '
|
||||
'was not available.';
|
||||
final user = 'Source: ${article.source.name}\n'
|
||||
'Title: ${article.title}\n'
|
||||
'URL: ${article.link}\n\n'
|
||||
'Article text:\n$text';
|
||||
return _send(system: system, user: user);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<DailyBriefing> createBriefing(
|
||||
List<Article> articles, SourceLanguage language) async {
|
||||
final input = BriefingInput(articles);
|
||||
final system = 'You are the editor of a daily tech news briefing. You receive '
|
||||
"today's headlines from several German and English tech news sites, each "
|
||||
'with an id in square brackets. Pick the 6-10 most important stories for a '
|
||||
'software developer interested in tech, merging coverage of the same story '
|
||||
'from different sources into one topic. Prefer substantial news (releases, '
|
||||
'security incidents, regulation, big business moves, AI) over deals, '
|
||||
'reviews and opinion pieces. Write all text in ${languageName(language)}. '
|
||||
'For each topic give a short title, a 2-4 sentence summary, and the ids of '
|
||||
'the articles it is based on. The intro is one or two sentences about the '
|
||||
"day's overall picture.";
|
||||
final user = "Today's headlines:\n\n${input.toPromptList()}";
|
||||
|
||||
final raw = await _send(system: system, user: user, schema: _briefingSchema);
|
||||
try {
|
||||
return input.toBriefing(jsonDecode(raw) as Map<String, dynamic>, name);
|
||||
} on FormatException {
|
||||
throw SummarizerException('Claude returned an unexpected briefing format.');
|
||||
}
|
||||
}
|
||||
|
||||
static const _briefingSchema = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'intro': {'type': 'string'},
|
||||
'topics': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'title': {'type': 'string'},
|
||||
'summary': {'type': 'string'},
|
||||
'article_ids': {
|
||||
'type': 'array',
|
||||
'items': {'type': 'string'},
|
||||
},
|
||||
},
|
||||
'required': ['title', 'summary', 'article_ids'],
|
||||
'additionalProperties': false,
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': ['intro', 'topics'],
|
||||
'additionalProperties': false,
|
||||
};
|
||||
|
||||
Future<String> _send({
|
||||
required String system,
|
||||
required String user,
|
||||
Map<String, dynamic>? schema,
|
||||
}) async {
|
||||
final useFallbacks = _fallbackModels.contains(model);
|
||||
final body = jsonEncode({
|
||||
'model': model,
|
||||
'max_tokens': 16000,
|
||||
'system': system,
|
||||
'messages': [
|
||||
{'role': 'user', 'content': user},
|
||||
],
|
||||
// Summarizing is a routine task; low effort keeps it fast and cheap.
|
||||
'output_config': {
|
||||
'effort': 'low',
|
||||
if (schema != null) 'format': {'type': 'json_schema', 'schema': schema},
|
||||
},
|
||||
// If a safety classifier declines, retry server-side on a fallback model.
|
||||
if (useFallbacks) 'fallbacks': 'default',
|
||||
});
|
||||
final headers = {
|
||||
'content-type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
if (useFallbacks) 'anthropic-beta': 'server-side-fallback-2026-07-01',
|
||||
if (kIsWeb) 'anthropic-dangerous-direct-browser-access': 'true',
|
||||
};
|
||||
|
||||
http.Response response;
|
||||
var attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
response = await _client
|
||||
.post(_endpoint, headers: headers, body: body)
|
||||
.timeout(const Duration(minutes: 3));
|
||||
} on Exception catch (e) {
|
||||
if (attempt++ < 2) {
|
||||
await Future.delayed(Duration(seconds: 2 * attempt));
|
||||
continue;
|
||||
}
|
||||
throw SummarizerException('Could not reach the Claude API: $e');
|
||||
}
|
||||
if (_retryStatus.contains(response.statusCode) && attempt++ < 2) {
|
||||
final retryAfter = int.tryParse(response.headers['retry-after'] ?? '');
|
||||
await Future.delayed(Duration(seconds: retryAfter ?? 3 * attempt));
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
final json = jsonDecode(utf8.decode(response.bodyBytes)) as Map<String, dynamic>;
|
||||
if (response.statusCode != 200) {
|
||||
final error = json['error'] as Map<String, dynamic>?;
|
||||
final message = error?['message'] ?? 'Unknown error';
|
||||
throw SummarizerException(switch (response.statusCode) {
|
||||
401 => 'Invalid Claude API key. Check it in Settings.',
|
||||
429 => 'Rate limited by the Claude API. Try again in a minute.',
|
||||
529 => 'The Claude API is overloaded right now. Try again shortly.',
|
||||
_ => 'Claude API error ${response.statusCode}: $message',
|
||||
});
|
||||
}
|
||||
|
||||
switch (json['stop_reason']) {
|
||||
case 'refusal':
|
||||
throw SummarizerException('Claude declined to summarize this content.');
|
||||
case 'max_tokens' when schema != null:
|
||||
throw SummarizerException('The briefing was cut off. Try fewer sources.');
|
||||
}
|
||||
|
||||
final text = (json['content'] as List)
|
||||
.where((block) => block['type'] == 'text')
|
||||
.map((block) => block['text'] as String)
|
||||
.join()
|
||||
.trim();
|
||||
if (text.isEmpty) throw SummarizerException('Claude returned an empty response.');
|
||||
return text;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
/// Gets past consent walls: sites that redirect to a separate consent page
|
||||
/// instead of showing the article (currently Golem).
|
||||
///
|
||||
/// Accepting makes the same request the site's own "accept" button makes and
|
||||
/// keeps the resulting cookies in memory, so both the reader view (via
|
||||
/// [cookieHeader]) and the in-app web view (via [cookiesFor]) can send them.
|
||||
/// Ads and trackers stay blocked separately.
|
||||
class ConsentService {
|
||||
ConsentService({http.Client? client}) : _client = client ?? http.Client();
|
||||
|
||||
final http.Client _client;
|
||||
|
||||
/// Domain (e.g. "golem.de") -> cookie name -> value.
|
||||
final Map<String, Map<String, String>> _cookies = {};
|
||||
|
||||
static final _walls = <_ConsentWall>[_GolemWall()];
|
||||
|
||||
static String domainOf(Uri url) {
|
||||
final parts = url.host.split('.');
|
||||
return parts.length <= 2 ? url.host : parts.sublist(parts.length - 2).join('.');
|
||||
}
|
||||
|
||||
/// True if [url] is a consent page this service knows how to accept.
|
||||
bool canAccept(Uri url) => _walls.any((w) => w.matches(url));
|
||||
|
||||
Map<String, String> cookiesFor(Uri url) => _cookies[domainOf(url)] ?? const {};
|
||||
|
||||
/// Value for a `Cookie` request header, or null if nothing was accepted.
|
||||
String? cookieHeader(Uri url) {
|
||||
final cookies = cookiesFor(url);
|
||||
if (cookies.isEmpty) return null;
|
||||
return cookies.entries.map((e) => '${e.key}=${e.value}').join('; ');
|
||||
}
|
||||
|
||||
/// Accepts the consent wall at [wallUrl]. [wallHtml] is the consent page,
|
||||
/// fetched if not given. Returns true if consent cookies were obtained.
|
||||
Future<bool> accept(Uri wallUrl, {String? wallHtml}) async {
|
||||
final wall = _walls.where((w) => w.matches(wallUrl)).firstOrNull;
|
||||
if (wall == null) return false;
|
||||
try {
|
||||
wallHtml ??= (await _client.get(wallUrl, headers: _headers)
|
||||
.timeout(const Duration(seconds: 15)))
|
||||
.body;
|
||||
final response = await _client
|
||||
.get(wall.acceptUrl(wallUrl, wallHtml), headers: _headers)
|
||||
.timeout(const Duration(seconds: 15));
|
||||
if (response.statusCode != 200) return false;
|
||||
final cookies = parseSetCookie(response.headers['set-cookie'] ?? '');
|
||||
if (!cookies.containsKey(wall.requiredCookie)) return false;
|
||||
_cookies.putIfAbsent(domainOf(wallUrl), () => {}).addAll(cookies);
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static const _headers = {'User-Agent': 'Mozilla/5.0 (DailyTechNews)'};
|
||||
|
||||
/// package:http joins multiple Set-Cookie headers with commas. Split only on
|
||||
/// commas that start a new `name=` pair, not those inside `expires=` dates.
|
||||
static Map<String, String> parseSetCookie(String header) {
|
||||
final cookies = <String, String>{};
|
||||
for (final cookie in header.split(RegExp(r',(?=\s*[^;,=\s]+=)'))) {
|
||||
final pair = cookie.split(';').first.trim();
|
||||
final eq = pair.indexOf('=');
|
||||
if (eq > 0) cookies[pair.substring(0, eq)] = pair.substring(eq + 1);
|
||||
}
|
||||
return cookies;
|
||||
}
|
||||
}
|
||||
|
||||
abstract class _ConsentWall {
|
||||
bool matches(Uri url);
|
||||
|
||||
/// The cookie that proves consent was stored.
|
||||
String get requiredCookie;
|
||||
|
||||
Uri acceptUrl(Uri wallUrl, String wallHtml);
|
||||
}
|
||||
|
||||
/// Golem redirects to /sonstiges/zustimmung/auswahl.html. Its consent script
|
||||
/// stores the choice by calling /abo/setconsentcookie.php; "simple" is the
|
||||
/// plain "Zustimmen" button Golem shows when its consent manager doesn't load.
|
||||
class _GolemWall implements _ConsentWall {
|
||||
static final _version = RegExp(r'"cookieVersion"\s*:\s*"([^"]+)"');
|
||||
|
||||
@override
|
||||
bool matches(Uri url) =>
|
||||
url.host.endsWith('golem.de') && url.path.startsWith('/sonstiges/zustimmung/');
|
||||
|
||||
@override
|
||||
String get requiredCookie => 'golem_consent20';
|
||||
|
||||
@override
|
||||
Uri acceptUrl(Uri wallUrl, String wallHtml) => Uri.https('www.golem.de', '/abo/setconsentcookie.php', {
|
||||
'type': 'simple',
|
||||
// Golem bumps this when its consent text changes; read it from the page.
|
||||
'version': _version.firstMatch(wallHtml)?.group(1) ?? '|250101',
|
||||
'expire': '1',
|
||||
'referer': wallUrl.queryParameters['from'] ?? 'https://www.golem.de/',
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import 'package:flutter_inappwebview/flutter_inappwebview.dart';
|
||||
|
||||
/// Ad, tracker and cookie-banner blocking for the in-app "original page" view.
|
||||
///
|
||||
/// Two layers: requests to known ad/tracker/consent hosts are blocked before
|
||||
/// they load, and a stylesheet injected at page start hides whatever slots and
|
||||
/// banners are left. Hiding a consent banner gives no consent, so sites treat
|
||||
/// it like a rejection.
|
||||
class ContentBlocking {
|
||||
/// Sites that redirect to a separate consent page instead of showing a
|
||||
/// banner. Nothing can be hidden there; it's accept, pay, or leave.
|
||||
static final consentWall = RegExp(r'golem\.de/sonstiges/zustimmung/|consent\.yahoo\.com/');
|
||||
|
||||
static const adHosts = [
|
||||
// Ad exchanges and networks
|
||||
'doubleclick.net', 'googlesyndication.com', 'googleadservices.com',
|
||||
'googletagservices.com', 'adservice.google.com', 'adservice.google.de',
|
||||
'amazon-adsystem.com', 'criteo.com', 'criteo.net', 'taboola.com', 'outbrain.com',
|
||||
'adnxs.com', 'rubiconproject.com', 'pubmatic.com', 'openx.net', 'casalemedia.com',
|
||||
'indexww.com', 'smartadserver.com', 'sascdn.com', 'yieldlab.net', 'adform.net',
|
||||
'adition.com', 'teads.tv', 'seedtag.com', '3lift.com', 'sharethrough.com',
|
||||
'33across.com', 'media.net', 'mgid.com', 'zemanta.com', 'yieldmo.com',
|
||||
'onetag-sys.com', 'bidswitch.net', 'lijit.com', 'sonobi.com', 'gumgum.com',
|
||||
'adsrvr.org', 'plista.com', 'ligatus.com', 'nativendo.de', 'traffective.com',
|
||||
'btloader.com', 'ad-delivery.net', 'adsafeprotected.com', 'moatads.com',
|
||||
'confiant-integrations.net', 'flashtalking.com', 'servedby-buysellads.com',
|
||||
// Tracking and analytics
|
||||
'google-analytics.com', 'googletagmanager.com', 'scorecardresearch.com',
|
||||
'chartbeat.com', 'chartbeat.net', 'ioam.de', 'iocnt.net', 'hotjar.com',
|
||||
'facebook.net', 'ads-twitter.com', 'analytics.tiktok.com', 'bat.bing.com',
|
||||
'quantserve.com', 'krxd.net', 'demdex.net', 'omtrdc.net', 'adobedtm.com',
|
||||
'permutive.com', 'permutive.app', 'cxense.com', 'id5-sync.com', 'rlcdn.com',
|
||||
'bluekai.com', 'everesttech.net', 'parsely.com', 'newrelic.com', 'nr-data.net',
|
||||
];
|
||||
|
||||
static const consentHosts = [
|
||||
'privacy-mgmt.com', 'sp-prod.net', 'sourcepoint.com', // Sourcepoint
|
||||
'cookielaw.org', 'onetrust.com', 'onetrust.io', // OneTrust
|
||||
'usercentrics.eu', 'usercentrics.com', 'privacy-center.org', 'didomi.io',
|
||||
'consensu.org', 'cookiebot.com', 'consentmanager.net', 'trustarc.com',
|
||||
'truste.com', 'cookieinformation.com', 'iubenda.com', 'cookie-script.com',
|
||||
'termly.io', 'osano.com', 'contentpass.net', 'contentpass.de', 'fundingchoicesmessages.google.com',
|
||||
];
|
||||
|
||||
static const _adSelectors = [
|
||||
'ins.adsbygoogle', '.adsbygoogle', '[id^="google_ads_iframe"]', '[id^="div-gpt-ad"]',
|
||||
'[data-google-query-id]', 'iframe[src*="doubleclick"]', '[id*="taboola"]',
|
||||
'[class*="taboola"]', '[id*="outbrain"]', '.OUTBRAIN', '[data-ad-slot]',
|
||||
'[data-ad-unit]', '.ad-container', '.ad-slot', '.ad-wrapper', '.ad-label',
|
||||
'.advertisement', '.advert', '.adzone', '.adunit', '.sponsored-content',
|
||||
'[aria-label="Advertisement"]', '[data-testid*="ad-slot"]',
|
||||
// Reserved slots that would otherwise stay as empty gaps (e.g. ComputerBase).
|
||||
'[class*="adbox"]', '[class*="adslot"]', '[class*="ad-slot"]', '[class*="ad_slot"]',
|
||||
'[id*="adslot"]', r'[class$="-ad"]', '[class*="-ad "]', '[class*="ad-container"]',
|
||||
];
|
||||
|
||||
static const _consentSelectors = [
|
||||
'[id^="sp_message_container"]', '.sp_veil', '#onetrust-consent-sdk',
|
||||
'#onetrust-banner-sdk', '.onetrust-pc-dark-filter', '#usercentrics-root',
|
||||
'#usercentrics-cmp-ui', '#didomi-host', '#CybotCookiebotDialog',
|
||||
'#CybotCookiebotDialogBodyUnderlay', '#cmpbox', '#cmpbox2', '.cmpboxBG',
|
||||
'#qc-cmp2-container', '.qc-cmp2-container', '#truste-consent-track',
|
||||
'.truste_overlay', '.truste_box_overlay', '#iubenda-cs-banner', '.fc-consent-root',
|
||||
'.fc-dialog-overlay', '#cookie-banner', '#cookiebanner', '#cookie-consent',
|
||||
'.cookie-banner', '.cookie-consent', '.cc-window', '.cc-banner',
|
||||
];
|
||||
|
||||
static String _hostPattern(List<String> hosts) =>
|
||||
r'^https?://([^/:]+\.)?(' + hosts.map(RegExp.escape).join('|') + r')([:/?]|$)';
|
||||
|
||||
/// Network-level blocking. One combined regex per category keeps the
|
||||
/// per-request check cheap.
|
||||
static List<ContentBlocker> blockers({required bool ads, required bool cookies}) => [
|
||||
if (ads) _block(_hostPattern(adHosts)),
|
||||
if (cookies) ...[
|
||||
_block(_hostPattern(consentHosts)),
|
||||
// Sourcepoint is often served from the site itself, e.g. cmp.heise.de.
|
||||
_block(r'^https?://cmp\.[^/]+/'),
|
||||
],
|
||||
];
|
||||
|
||||
static ContentBlocker _block(String pattern) => ContentBlocker(
|
||||
trigger: ContentBlockerTrigger(urlFilter: pattern),
|
||||
action: ContentBlockerAction(type: ContentBlockerActionType.BLOCK),
|
||||
);
|
||||
|
||||
/// A script for page start that injects the hiding stylesheet and keeps it
|
||||
/// in place. Returns null when nothing is enabled.
|
||||
static String? cosmeticScript({required bool ads, required bool cookies}) {
|
||||
final selectors = [if (ads) ..._adSelectors, if (cookies) ..._consentSelectors];
|
||||
if (selectors.isEmpty) return null;
|
||||
final css = StringBuffer('${selectors.join(',')}{display:none!important}');
|
||||
if (cookies) {
|
||||
// Consent dialogs lock scrolling on <html>/<body>; undo that.
|
||||
css.write('html,body{overflow:auto!important;overflow-y:auto!important}');
|
||||
}
|
||||
final cssLiteral = css.toString().replaceAll(r'\', r'\\').replaceAll("'", r"\'");
|
||||
return '''
|
||||
(function () {
|
||||
var css = '$cssLiteral';
|
||||
var unlockConsent = $cookies;
|
||||
var consentPattern = /consent|cookie|cmp|gdpr|sp-message|didomi|onetrust|usercentrics/i;
|
||||
function apply() {
|
||||
if (document.getElementById('tagesbyte-blocker')) return;
|
||||
var style = document.createElement('style');
|
||||
style.id = 'tagesbyte-blocker';
|
||||
style.textContent = css;
|
||||
(document.head || document.documentElement).appendChild(style);
|
||||
}
|
||||
// Some sites (e.g. ComputerBase) build their own consent wall as a modal
|
||||
// <dialog>, which makes the rest of the page inert even when hidden.
|
||||
function unlock() {
|
||||
if (!unlockConsent) return;
|
||||
document.querySelectorAll('dialog').forEach(function (d) {
|
||||
if (consentPattern.test(d.className + ' ' + d.id)) {
|
||||
try { d.close(); } catch (e) {}
|
||||
d.remove();
|
||||
}
|
||||
});
|
||||
[document.documentElement, document.body].forEach(function (el) {
|
||||
if (!el) return;
|
||||
Array.prototype.slice.call(el.classList).forEach(function (c) {
|
||||
if (consentPattern.test(c)) el.classList.remove(c);
|
||||
});
|
||||
});
|
||||
}
|
||||
apply();
|
||||
document.addEventListener('DOMContentLoaded', function () { apply(); unlock(); });
|
||||
window.addEventListener('load', function () { apply(); unlock(); });
|
||||
// Consent scripts often open their dialog a moment after load.
|
||||
var runs = 0;
|
||||
var timer = setInterval(function () { unlock(); if (++runs > 30) clearInterval(timer); }, 500);
|
||||
})();
|
||||
''';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import 'package:html/dom.dart' as dom;
|
||||
import 'package:html/parser.dart' as html_parser;
|
||||
import 'package:xml/xml.dart';
|
||||
|
||||
import '../models/article.dart';
|
||||
import '../models/news_source.dart';
|
||||
|
||||
/// Parses RSS 2.0, RSS 1.0 (RDF) and Atom feeds into [Article]s.
|
||||
class FeedParser {
|
||||
static const excerptLength = 400;
|
||||
|
||||
static List<Article> parse(String body, NewsSource source) {
|
||||
final doc = XmlDocument.parse(body);
|
||||
final root = doc.rootElement;
|
||||
if (root.name.local == 'feed') {
|
||||
return root.childElements
|
||||
.where((e) => e.name.local == 'entry')
|
||||
.map((e) => _parseAtomEntry(e, source))
|
||||
.whereType<Article>()
|
||||
.toList();
|
||||
}
|
||||
// RSS 2.0 nests <item> in <channel>, RSS 1.0 puts them at the root.
|
||||
return root.descendantElements
|
||||
.where((e) => e.name.local == 'item')
|
||||
.map((e) => _parseRssItem(e, source))
|
||||
.whereType<Article>()
|
||||
.toList();
|
||||
}
|
||||
|
||||
static Article? _parseRssItem(XmlElement item, NewsSource source) {
|
||||
final title = htmlToText(_text(item, 'title'));
|
||||
final link = _text(item, 'link').trim().isNotEmpty
|
||||
? _text(item, 'link').trim()
|
||||
: _text(item, 'guid').trim();
|
||||
if (title.isEmpty || !link.startsWith('http')) return null;
|
||||
|
||||
final descriptionHtml = _text(item, 'description');
|
||||
final contentHtml = _text(item, 'encoded', prefix: 'content');
|
||||
final date = _text(item, 'pubDate').isNotEmpty
|
||||
? _text(item, 'pubDate')
|
||||
: _text(item, 'date', prefix: 'dc');
|
||||
|
||||
return Article(
|
||||
title: title,
|
||||
link: link,
|
||||
source: source,
|
||||
published: parseDate(date),
|
||||
excerpt: _excerpt(descriptionHtml.isNotEmpty ? descriptionHtml : contentHtml),
|
||||
contentHtml: contentHtml,
|
||||
imageUrl: _imageFrom(item) ?? _firstImg(contentHtml) ?? _firstImg(descriptionHtml),
|
||||
author: _nonEmpty(htmlToText(_text(item, 'creator', prefix: 'dc'))) ??
|
||||
_nonEmpty(_text(item, 'author')),
|
||||
);
|
||||
}
|
||||
|
||||
static Article? _parseAtomEntry(XmlElement entry, NewsSource source) {
|
||||
final title = htmlToText(_text(entry, 'title'));
|
||||
final links = entry.childElements.where((e) => e.name.local == 'link');
|
||||
final link = links
|
||||
.where((l) => (l.getAttribute('rel') ?? 'alternate') == 'alternate')
|
||||
.map((l) => l.getAttribute('href'))
|
||||
.firstWhere((h) => h != null, orElse: () => null) ??
|
||||
links.map((l) => l.getAttribute('href')).firstWhere((h) => h != null, orElse: () => null);
|
||||
if (title.isEmpty || link == null || !link.startsWith('http')) return null;
|
||||
|
||||
final summaryHtml = _text(entry, 'summary');
|
||||
final contentHtml = _text(entry, 'content');
|
||||
final date = _text(entry, 'published').isNotEmpty
|
||||
? _text(entry, 'published')
|
||||
: _text(entry, 'updated');
|
||||
final authorEl = _child(entry, 'author');
|
||||
|
||||
return Article(
|
||||
title: title,
|
||||
link: link,
|
||||
source: source,
|
||||
published: parseDate(date),
|
||||
excerpt: _excerpt(summaryHtml.isNotEmpty ? summaryHtml : contentHtml),
|
||||
contentHtml: contentHtml,
|
||||
imageUrl: _imageFrom(entry) ?? _firstImg(contentHtml) ?? _firstImg(summaryHtml),
|
||||
author: authorEl == null ? null : _nonEmpty(_text(authorEl, 'name')),
|
||||
);
|
||||
}
|
||||
|
||||
static XmlElement? _child(XmlElement parent, String local, {String? prefix}) {
|
||||
for (final e in parent.childElements) {
|
||||
if (e.name.local == local && (prefix == null || e.name.prefix == prefix)) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String _text(XmlElement parent, String local, {String? prefix}) =>
|
||||
_child(parent, local, prefix: prefix)?.innerText.trim() ?? '';
|
||||
|
||||
static String? _nonEmpty(String s) => s.trim().isEmpty ? null : s.trim();
|
||||
|
||||
static String? _imageFrom(XmlElement item) {
|
||||
for (final e in item.descendantElements) {
|
||||
final local = e.name.local;
|
||||
final url = e.getAttribute('url');
|
||||
if (url == null) continue;
|
||||
if (local == 'thumbnail') return url;
|
||||
if (local == 'content' && e.name.prefix == 'media') {
|
||||
final medium = e.getAttribute('medium') ?? e.getAttribute('type') ?? 'image';
|
||||
if (medium.startsWith('image')) return url;
|
||||
}
|
||||
if (local == 'enclosure' && (e.getAttribute('type') ?? '').startsWith('image')) {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String? _firstImg(String html) {
|
||||
if (!html.contains('<img')) return null;
|
||||
for (final img in html_parser.parseFragment(html).querySelectorAll('img')) {
|
||||
// Skip tracking pixels (e.g. Golem's 1x1 counter image) and icons.
|
||||
final width = int.tryParse(img.attributes['width'] ?? '');
|
||||
final height = int.tryParse(img.attributes['height'] ?? '');
|
||||
if ((width != null && width < 100) || (height != null && height < 100)) continue;
|
||||
final src = img.attributes['src'];
|
||||
if (src != null && src.startsWith('http')) return src;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String _excerpt(String html) {
|
||||
final text = htmlToText(html);
|
||||
// Some feeds (e.g. Hacker News) only carry a "Comments" link here.
|
||||
if (text.length < 20) return '';
|
||||
if (text.length <= excerptLength) return text;
|
||||
final cut = text.substring(0, excerptLength);
|
||||
final lastSpace = cut.lastIndexOf(' ');
|
||||
return '${cut.substring(0, lastSpace > 0 ? lastSpace : excerptLength)}…';
|
||||
}
|
||||
|
||||
/// Converts an HTML snippet to whitespace-normalized plain text.
|
||||
static String htmlToText(String html) {
|
||||
if (html.isEmpty) return '';
|
||||
final fragment = html_parser.parseFragment(html);
|
||||
fragment.querySelectorAll('script, style').forEach((dom.Element e) => e.remove());
|
||||
return (fragment.text ?? '').replaceAll(RegExp(r'\s+'), ' ').trim();
|
||||
}
|
||||
|
||||
static const _months = {
|
||||
'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'may': 5, 'jun': 6,
|
||||
'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12,
|
||||
};
|
||||
|
||||
static const _zones = {
|
||||
'UT': 0, 'GMT': 0, 'Z': 0, 'UTC': 0,
|
||||
'EST': -5, 'EDT': -4, 'CST': -6, 'CDT': -5,
|
||||
'MST': -7, 'MDT': -6, 'PST': -8, 'PDT': -7,
|
||||
'CET': 1, 'CEST': 2,
|
||||
};
|
||||
|
||||
static final _rfc822 = RegExp(
|
||||
r'^(?:\w{3},\s*)?(\d{1,2})\s+(\w{3})\w*\s+(\d{2,4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*([+-]\d{4}|[A-Za-z]+)?',
|
||||
);
|
||||
|
||||
/// Parses ISO 8601 (Atom) and RFC 822 (RSS) dates. Returns UTC.
|
||||
static DateTime? parseDate(String raw) {
|
||||
final s = raw.trim();
|
||||
if (s.isEmpty) return null;
|
||||
final iso = DateTime.tryParse(s);
|
||||
if (iso != null) return iso.toUtc();
|
||||
|
||||
final m = _rfc822.firstMatch(s);
|
||||
if (m == null) return null;
|
||||
final month = _months[m.group(2)!.toLowerCase()];
|
||||
if (month == null) return null;
|
||||
var year = int.parse(m.group(3)!);
|
||||
if (year < 100) year += 2000;
|
||||
final utc = DateTime.utc(
|
||||
year,
|
||||
month,
|
||||
int.parse(m.group(1)!),
|
||||
int.parse(m.group(4)!),
|
||||
int.parse(m.group(5)!),
|
||||
int.parse(m.group(6) ?? '0'),
|
||||
);
|
||||
|
||||
final zone = m.group(7);
|
||||
var offset = Duration.zero;
|
||||
if (zone != null && (zone.startsWith('+') || zone.startsWith('-'))) {
|
||||
final sign = zone.startsWith('-') ? -1 : 1;
|
||||
offset = Duration(
|
||||
hours: int.parse(zone.substring(1, 3)),
|
||||
minutes: int.parse(zone.substring(3, 5)),
|
||||
) * sign;
|
||||
} else if (zone != null) {
|
||||
offset = Duration(hours: _zones[zone.toUpperCase()] ?? 0);
|
||||
}
|
||||
return utc.subtract(offset);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
import '../models/article.dart';
|
||||
import '../models/news_source.dart';
|
||||
import 'feed_parser.dart';
|
||||
|
||||
class FeedResult {
|
||||
const FeedResult(this.articles, this.errors);
|
||||
|
||||
final List<Article> articles;
|
||||
|
||||
/// Source name -> error message for feeds that failed to load.
|
||||
final Map<String, String> errors;
|
||||
}
|
||||
|
||||
class FeedService {
|
||||
FeedService({http.Client? client}) : _client = client ?? http.Client();
|
||||
|
||||
final http.Client _client;
|
||||
|
||||
static const _timeout = Duration(seconds: 20);
|
||||
|
||||
/// Fetches all [sources] in parallel. Failing feeds are reported in
|
||||
/// [FeedResult.errors] instead of failing the whole refresh.
|
||||
///
|
||||
/// [corsProxy] is only used on web, where browsers block cross-origin feed
|
||||
/// requests. It is prepended to the feed URL, e.g. `https://proxy/?url=`.
|
||||
Future<FeedResult> fetchAll(
|
||||
List<NewsSource> sources, {
|
||||
required Duration maxAge,
|
||||
String corsProxy = '',
|
||||
}) async {
|
||||
final errors = <String, String>{};
|
||||
final results = await Future.wait(sources.map((source) async {
|
||||
try {
|
||||
return await fetchSource(source, corsProxy: corsProxy);
|
||||
} catch (e) {
|
||||
errors[source.name] = _describe(e);
|
||||
return <Article>[];
|
||||
}
|
||||
}));
|
||||
|
||||
final cutoff = DateTime.now().toUtc().subtract(maxAge);
|
||||
final seen = <String>{};
|
||||
final articles = <Article>[
|
||||
for (final list in results)
|
||||
for (final a in list)
|
||||
if ((a.published == null || a.published!.isAfter(cutoff)) && seen.add(a.id)) a,
|
||||
];
|
||||
articles.sort((a, b) {
|
||||
final pa = a.published, pb = b.published;
|
||||
if (pa == null && pb == null) return 0;
|
||||
if (pa == null) return 1;
|
||||
if (pb == null) return -1;
|
||||
return pb.compareTo(pa);
|
||||
});
|
||||
return FeedResult(articles, errors);
|
||||
}
|
||||
|
||||
Future<List<Article>> fetchSource(NewsSource source, {String corsProxy = ''}) async {
|
||||
final url = kIsWeb && corsProxy.isNotEmpty
|
||||
? '$corsProxy${Uri.encodeComponent(source.feedUrl)}'
|
||||
: source.feedUrl;
|
||||
final response = await _client.get(
|
||||
Uri.parse(url),
|
||||
headers: {
|
||||
'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*',
|
||||
if (!kIsWeb) 'User-Agent': 'DailyTechNews/0.1 (Flutter RSS reader)',
|
||||
},
|
||||
).timeout(_timeout);
|
||||
if (response.statusCode != 200) {
|
||||
throw Exception('HTTP ${response.statusCode}');
|
||||
}
|
||||
return FeedParser.parse(decodeBody(response.bodyBytes), source);
|
||||
}
|
||||
|
||||
/// Decodes feed bytes honoring the XML declaration's encoding; defaults to UTF-8.
|
||||
static String decodeBody(List<int> bytes) {
|
||||
final head = latin1.decode(bytes.take(200).toList()).toLowerCase();
|
||||
final match = RegExp(r'''encoding=["']([\w-]+)["']''').firstMatch(head);
|
||||
final encoding = match?.group(1);
|
||||
if (encoding == 'iso-8859-1' || encoding == 'latin1' || encoding == 'windows-1252') {
|
||||
return latin1.decode(bytes);
|
||||
}
|
||||
return utf8.decode(bytes, allowMalformed: true);
|
||||
}
|
||||
|
||||
static String _describe(Object e) {
|
||||
final text = e.toString();
|
||||
if (kIsWeb && text.contains('ClientException')) {
|
||||
return 'Blocked by the browser (CORS). Set a CORS proxy in Settings.';
|
||||
}
|
||||
return text.replaceFirst('Exception: ', '');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
import '../models/article.dart';
|
||||
import '../models/briefing.dart';
|
||||
import '../models/news_source.dart';
|
||||
import 'summarizer.dart';
|
||||
|
||||
/// Delegates summarizing to an existing n8n workflow via a Webhook trigger.
|
||||
/// The request/response contract is documented in README.md.
|
||||
class N8nSummarizer implements Summarizer {
|
||||
N8nSummarizer({required this.webhookUrl, http.Client? client})
|
||||
: _client = client ?? http.Client();
|
||||
|
||||
final String webhookUrl;
|
||||
final http.Client _client;
|
||||
|
||||
@override
|
||||
String get name => 'n8n workflow';
|
||||
|
||||
@override
|
||||
Future<String> summarizeArticle(
|
||||
Article article, String text, SourceLanguage language) async {
|
||||
final body = await _post({
|
||||
'action': 'summarize_article',
|
||||
'language': language.name,
|
||||
'article': {
|
||||
'title': article.title,
|
||||
'url': article.link,
|
||||
'source': article.source.name,
|
||||
'published': article.published?.toIso8601String(),
|
||||
'text': text,
|
||||
},
|
||||
});
|
||||
if (body is Map && body['summary'] is String) return body['summary'] as String;
|
||||
if (body is String && body.trim().isNotEmpty) return body.trim();
|
||||
throw SummarizerException('n8n response has no "summary" field.');
|
||||
}
|
||||
|
||||
@override
|
||||
Future<DailyBriefing> createBriefing(
|
||||
List<Article> articles, SourceLanguage language) async {
|
||||
final input = BriefingInput(articles);
|
||||
final body = await _post({
|
||||
'action': 'daily_briefing',
|
||||
'language': language.name,
|
||||
'articles': input.toJsonList(),
|
||||
});
|
||||
if (body is Map<String, dynamic> && body['topics'] is List) {
|
||||
return input.toBriefing(body, name);
|
||||
}
|
||||
// Accept a plain text digest (e.g. the message the workflow posts to Discord).
|
||||
final text = body is Map ? body['text'] ?? body['summary'] : body;
|
||||
if (text is String && text.trim().isNotEmpty) {
|
||||
return DailyBriefing(
|
||||
createdAt: DateTime.now(),
|
||||
intro: text.trim(),
|
||||
topics: const [],
|
||||
generatedBy: name,
|
||||
);
|
||||
}
|
||||
throw SummarizerException('n8n response has neither "topics" nor "text".');
|
||||
}
|
||||
|
||||
Future<Object?> _post(Map<String, dynamic> payload) async {
|
||||
final http.Response response;
|
||||
try {
|
||||
response = await _client
|
||||
.post(
|
||||
Uri.parse(webhookUrl),
|
||||
headers: {'content-type': 'application/json'},
|
||||
body: jsonEncode(payload),
|
||||
)
|
||||
.timeout(const Duration(minutes: 3));
|
||||
} on Exception catch (e) {
|
||||
throw SummarizerException('Could not reach the n8n webhook: $e');
|
||||
}
|
||||
if (response.statusCode != 200) {
|
||||
throw SummarizerException('n8n webhook returned HTTP ${response.statusCode}.');
|
||||
}
|
||||
final text = utf8.decode(response.bodyBytes);
|
||||
try {
|
||||
final json = jsonDecode(text);
|
||||
// "Respond to Webhook" with "All Incoming Items" wraps results in a list.
|
||||
return json is List && json.isNotEmpty ? json.first : json;
|
||||
} on FormatException {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
import '../models/article.dart';
|
||||
import 'storage.dart';
|
||||
|
||||
/// Recognizes paid and sponsored articles so the feed can hide them.
|
||||
///
|
||||
/// Most paid articles are marked in the title (heise+, Golem's "(g+)").
|
||||
/// SPIEGEL and t3n only mark them in the page's schema.org data
|
||||
/// (`"isAccessibleForFree": false`), so for those sources each new article is
|
||||
/// checked once in the background and the result is remembered.
|
||||
class PaywallService {
|
||||
PaywallService(this._storage, {http.Client? client})
|
||||
: _client = client ?? http.Client(),
|
||||
_checks = _storage.loadPaywallChecks();
|
||||
|
||||
final Storage _storage;
|
||||
final http.Client _client;
|
||||
|
||||
/// Article URL -> (paid, checked at).
|
||||
final Map<String, ({bool paid, DateTime checkedAt})> _checks;
|
||||
|
||||
static final paidTitle = RegExp(
|
||||
r'^\s*(heise\+|\(g\+\)|SPIEGEL\+|Z\+|t3n Pro\b|c.t\+)',
|
||||
caseSensitive: false,
|
||||
);
|
||||
|
||||
static final sponsoredTitle = RegExp(
|
||||
r'^\s*(Anzeige|Sponsored|Advertorial|Werbung|Promotion)\s*[:|]',
|
||||
caseSensitive: false,
|
||||
);
|
||||
|
||||
static final _marker =
|
||||
RegExp(r'''"isAccessibleForFree"\s*:\s*"?(true|false)"?''', caseSensitive: false);
|
||||
|
||||
/// Stop reading a page after this much without finding the marker.
|
||||
static const _maxBytes = 1500000;
|
||||
static const _concurrency = 6;
|
||||
static const _keepFor = Duration(days: 14);
|
||||
|
||||
bool isPaid(Article a) => paidTitle.hasMatch(a.title) || (_checks[a.link]?.paid ?? false);
|
||||
|
||||
bool isSponsored(Article a) => sponsoredTitle.hasMatch(a.title);
|
||||
|
||||
/// Checks articles from page-checked sources that haven't been seen yet.
|
||||
/// Returns true if any new paid article was found.
|
||||
Future<bool> checkPages(Iterable<Article> articles) async {
|
||||
final queue = articles
|
||||
.where((a) =>
|
||||
a.source.checkPaywallOnPage &&
|
||||
!_checks.containsKey(a.link) &&
|
||||
!paidTitle.hasMatch(a.title))
|
||||
.toList();
|
||||
if (queue.isEmpty) return false;
|
||||
|
||||
var foundPaid = false;
|
||||
Future<void> worker() async {
|
||||
while (queue.isNotEmpty) {
|
||||
final article = queue.removeLast();
|
||||
final paid = await _fetchIsPaid(article.link);
|
||||
if (paid == null) continue; // network error: try again next refresh
|
||||
_checks[article.link] = (paid: paid, checkedAt: DateTime.now());
|
||||
foundPaid |= paid;
|
||||
}
|
||||
}
|
||||
|
||||
await Future.wait(List.generate(_concurrency, (_) => worker()));
|
||||
final cutoff = DateTime.now().subtract(_keepFor);
|
||||
_checks.removeWhere((_, v) => v.checkedAt.isBefore(cutoff));
|
||||
await _storage.savePaywallChecks(_checks);
|
||||
return foundPaid;
|
||||
}
|
||||
|
||||
/// Streams the page and stops as soon as the schema.org marker shows up.
|
||||
/// Returns null on network errors, false if the page has no marker.
|
||||
Future<bool?> _fetchIsPaid(String url) async {
|
||||
try {
|
||||
final request = http.Request('GET', Uri.parse(url))
|
||||
..headers['User-Agent'] = 'Mozilla/5.0 (DailyTechNews)';
|
||||
final response = await _client.send(request).timeout(const Duration(seconds: 15));
|
||||
if (response.statusCode != 200) {
|
||||
await response.stream.listen(null).cancel();
|
||||
return null;
|
||||
}
|
||||
var tail = '';
|
||||
var read = 0;
|
||||
await for (final chunk in response.stream.timeout(const Duration(seconds: 15))) {
|
||||
read += chunk.length;
|
||||
// The marker is ASCII, so latin1 avoids errors on split UTF-8 sequences.
|
||||
// Keep a little overlap in case the marker spans two chunks.
|
||||
final text = tail + latin1.decode(chunk);
|
||||
final match = _marker.firstMatch(text);
|
||||
if (match != null) return match.group(1)!.toLowerCase() == 'false';
|
||||
if (read > _maxBytes) break;
|
||||
tail = text.length > 64 ? text.substring(text.length - 64) : text;
|
||||
}
|
||||
return false;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter/foundation.dart' show kIsWeb;
|
||||
import 'package:flutter/material.dart' show ThemeMode;
|
||||
import 'package:flutter_secure_storage/flutter_secure_storage.dart';
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
|
||||
import '../models/app_settings.dart';
|
||||
import '../models/briefing.dart';
|
||||
import '../models/news_source.dart';
|
||||
|
||||
/// Persists settings, source selection and the last briefing.
|
||||
/// The Claude API key goes to the platform keystore, everything else to
|
||||
/// shared preferences.
|
||||
class Storage {
|
||||
Storage(this._prefs);
|
||||
|
||||
static Future<Storage> open() async => Storage(await SharedPreferences.getInstance());
|
||||
|
||||
final SharedPreferences _prefs;
|
||||
final _secure = const FlutterSecureStorage();
|
||||
|
||||
static const _apiKey = 'claude_api_key';
|
||||
|
||||
Future<AppSettings> loadSettings() async {
|
||||
String apiKey = '';
|
||||
try {
|
||||
apiKey = await _secure.read(key: _apiKey) ?? '';
|
||||
} catch (_) {
|
||||
// Keystore unavailable (e.g. some web/desktop setups) -> user re-enters it.
|
||||
}
|
||||
return AppSettings(
|
||||
aiProvider: AiProvider.values.byName(_prefs.getString('ai_provider') ?? 'none'),
|
||||
claudeApiKey: apiKey,
|
||||
claudeModel: _prefs.getString('claude_model') ?? 'claude-opus-5-5',
|
||||
n8nWebhookUrl: _prefs.getString('n8n_webhook_url') ?? '',
|
||||
summaryLanguage:
|
||||
SourceLanguage.values.byName(_prefs.getString('summary_language') ?? 'de'),
|
||||
maxAgeHours: _prefs.getInt('max_age_hours') ?? 24,
|
||||
corsProxy: _prefs.getString('cors_proxy') ?? '',
|
||||
hidePaid: _prefs.getBool('hide_paid') ?? true,
|
||||
hideSponsored: _prefs.getBool('hide_sponsored') ?? true,
|
||||
blockAds: _prefs.getBool('block_ads') ?? true,
|
||||
hideCookieBanners: _prefs.getBool('hide_cookie_banners') ?? true,
|
||||
acceptConsentWalls: _prefs.getBool('accept_consent_walls') ?? true,
|
||||
themeMode: ThemeMode.values.asNameMap()[_prefs.getString('theme_mode')] ?? ThemeMode.system,
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> saveSettings(AppSettings s) async {
|
||||
await _prefs.setString('ai_provider', s.aiProvider.name);
|
||||
await _prefs.setString('claude_model', s.claudeModel);
|
||||
await _prefs.setString('n8n_webhook_url', s.n8nWebhookUrl);
|
||||
await _prefs.setString('summary_language', s.summaryLanguage.name);
|
||||
await _prefs.setInt('max_age_hours', s.maxAgeHours);
|
||||
await _prefs.setString('cors_proxy', s.corsProxy);
|
||||
await _prefs.setBool('hide_paid', s.hidePaid);
|
||||
await _prefs.setBool('hide_sponsored', s.hideSponsored);
|
||||
await _prefs.setBool('block_ads', s.blockAds);
|
||||
await _prefs.setBool('hide_cookie_banners', s.hideCookieBanners);
|
||||
await _prefs.setBool('accept_consent_walls', s.acceptConsentWalls);
|
||||
await _prefs.setString('theme_mode', s.themeMode.name);
|
||||
try {
|
||||
await _secure.write(key: _apiKey, value: s.claudeApiKey);
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
/// Null means "never configured" -> use each source's default.
|
||||
Set<String>? loadEnabledSourceIds() =>
|
||||
_prefs.getStringList('enabled_sources')?.toSet();
|
||||
|
||||
Future<void> saveEnabledSourceIds(Set<String> ids) =>
|
||||
_prefs.setStringList('enabled_sources', ids.toList());
|
||||
|
||||
List<NewsSource> loadCustomSources() {
|
||||
final raw = _prefs.getString('custom_sources');
|
||||
if (raw == null) return [];
|
||||
return (jsonDecode(raw) as List)
|
||||
.map((j) => NewsSource.custom(j as Map<String, dynamic>))
|
||||
.toList();
|
||||
}
|
||||
|
||||
Future<void> saveCustomSources(List<NewsSource> sources) => _prefs.setString(
|
||||
'custom_sources', jsonEncode(sources.map((s) => s.toJson()).toList()));
|
||||
|
||||
/// Article URL -> whether its page said it's paid, and when that was checked.
|
||||
Map<String, ({bool paid, DateTime checkedAt})> loadPaywallChecks() {
|
||||
final raw = _prefs.getString('paywall_checks');
|
||||
if (raw == null) return {};
|
||||
try {
|
||||
return (jsonDecode(raw) as Map<String, dynamic>).map((url, v) {
|
||||
final entry = v as List;
|
||||
return MapEntry(url, (
|
||||
paid: entry[0] as bool,
|
||||
checkedAt: DateTime.fromMillisecondsSinceEpoch(entry[1] as int),
|
||||
));
|
||||
});
|
||||
} catch (_) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> savePaywallChecks(Map<String, ({bool paid, DateTime checkedAt})> checks) =>
|
||||
_prefs.setString(
|
||||
'paywall_checks',
|
||||
jsonEncode(checks.map(
|
||||
(url, v) => MapEntry(url, [v.paid, v.checkedAt.millisecondsSinceEpoch]))),
|
||||
);
|
||||
|
||||
/// All saved briefings, newest first. Stored as a JSON file in the app's
|
||||
/// private storage (preferences on web), since the history keeps growing.
|
||||
Future<List<DailyBriefing>> loadBriefingHistory() async {
|
||||
List<DailyBriefing> history;
|
||||
try {
|
||||
String? raw;
|
||||
if (kIsWeb) {
|
||||
raw = _prefs.getString(_historyKey);
|
||||
} else {
|
||||
final file = await _historyFile();
|
||||
if (await file.exists()) raw = await file.readAsString();
|
||||
}
|
||||
history = raw == null
|
||||
? []
|
||||
: (jsonDecode(raw) as List)
|
||||
.map((j) => DailyBriefing.fromJson(j as Map<String, dynamic>))
|
||||
.toList();
|
||||
} catch (_) {
|
||||
history = [];
|
||||
}
|
||||
|
||||
// Earlier versions kept only the last briefing in preferences.
|
||||
final legacy = _prefs.getString('last_briefing');
|
||||
if (legacy != null) {
|
||||
try {
|
||||
final briefing = DailyBriefing.fromJson(jsonDecode(legacy) as Map<String, dynamic>);
|
||||
if (!history.any((b) => b.createdAt == briefing.createdAt)) history.add(briefing);
|
||||
await saveBriefingHistory(history);
|
||||
} catch (_) {}
|
||||
await _prefs.remove('last_briefing');
|
||||
}
|
||||
|
||||
history.sort((a, b) => b.createdAt.compareTo(a.createdAt));
|
||||
return history;
|
||||
}
|
||||
|
||||
Future<void> saveBriefingHistory(List<DailyBriefing> history) async {
|
||||
final json = jsonEncode(history.map((b) => b.toJson()).toList());
|
||||
if (kIsWeb) {
|
||||
await _prefs.setString(_historyKey, json);
|
||||
return;
|
||||
}
|
||||
// Write to a temp file and rename, so a crash mid-write can't corrupt it.
|
||||
final file = await _historyFile();
|
||||
final temp = File('${file.path}.tmp');
|
||||
await temp.writeAsString(json, flush: true);
|
||||
await temp.rename(file.path);
|
||||
}
|
||||
|
||||
static const _historyKey = 'briefing_history';
|
||||
|
||||
Future<File> _historyFile() async =>
|
||||
File('${(await getApplicationSupportDirectory()).path}/briefings.json');
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import '../models/article.dart';
|
||||
import '../models/briefing.dart';
|
||||
import '../models/news_source.dart';
|
||||
|
||||
class SummarizerException implements Exception {
|
||||
SummarizerException(this.message);
|
||||
|
||||
final String message;
|
||||
|
||||
@override
|
||||
String toString() => message;
|
||||
}
|
||||
|
||||
/// Something that can turn articles into summaries. Implemented by the
|
||||
/// direct Claude API client and by the n8n webhook bridge.
|
||||
abstract class Summarizer {
|
||||
String get name;
|
||||
|
||||
Future<String> summarizeArticle(Article article, String text, SourceLanguage language);
|
||||
|
||||
Future<DailyBriefing> createBriefing(List<Article> articles, SourceLanguage language);
|
||||
}
|
||||
|
||||
/// Shared prompt material and response mapping for all summarizers.
|
||||
class BriefingInput {
|
||||
BriefingInput(List<Article> articles)
|
||||
: byId = {for (var i = 0; i < articles.length; i++) 'a${i + 1}': articles[i]};
|
||||
|
||||
/// Short ids keep the prompt compact and are easy for the model to cite.
|
||||
final Map<String, Article> byId;
|
||||
|
||||
static const excerptChars = 220;
|
||||
|
||||
String toPromptList() {
|
||||
final buffer = StringBuffer();
|
||||
byId.forEach((id, a) {
|
||||
final excerpt = a.excerpt.length > excerptChars
|
||||
? '${a.excerpt.substring(0, excerptChars)}…'
|
||||
: a.excerpt;
|
||||
buffer.writeln('[$id] (${a.source.name}) ${a.title}'
|
||||
'${excerpt.isEmpty ? '' : ' — $excerpt'}');
|
||||
});
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
List<Map<String, dynamic>> toJsonList() => [
|
||||
for (final e in byId.entries)
|
||||
{
|
||||
'id': e.key,
|
||||
'title': e.value.title,
|
||||
'url': e.value.link,
|
||||
'source': e.value.source.name,
|
||||
'language': e.value.source.language.name,
|
||||
'published': e.value.published?.toIso8601String(),
|
||||
'excerpt': e.value.excerpt,
|
||||
}
|
||||
];
|
||||
|
||||
/// Maps `{intro, topics: [{title, summary, article_ids}]}` onto a briefing.
|
||||
DailyBriefing toBriefing(Map<String, dynamic> json, String generatedBy) {
|
||||
final topics = (json['topics'] as List? ?? []).map((t) {
|
||||
final topic = t as Map<String, dynamic>;
|
||||
final ids = (topic['article_ids'] as List? ?? []).cast<String>();
|
||||
return BriefingTopic(
|
||||
title: topic['title'] as String? ?? '',
|
||||
summary: topic['summary'] as String? ?? '',
|
||||
sources: [
|
||||
for (final id in ids)
|
||||
if (byId[id] != null)
|
||||
ArticleRef(
|
||||
title: byId[id]!.title,
|
||||
url: byId[id]!.link,
|
||||
sourceName: byId[id]!.source.name,
|
||||
),
|
||||
],
|
||||
);
|
||||
}).toList();
|
||||
return DailyBriefing(
|
||||
createdAt: DateTime.now(),
|
||||
intro: json['intro'] as String? ?? '',
|
||||
topics: topics,
|
||||
generatedBy: generatedBy,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
String languageName(SourceLanguage language) =>
|
||||
language == SourceLanguage.de ? 'German' : 'English';
|
||||
@@ -0,0 +1,26 @@
|
||||
/// Dark mode for the in-app "original page" view.
|
||||
///
|
||||
/// When the phone itself is in dark mode, Android darkens pages on its own
|
||||
/// (algorithmic darkening, plus `prefers-color-scheme: dark` for sites with a
|
||||
/// dark design). When only the app is set to dark, Android doesn't, so this
|
||||
/// stylesheet inverts the page and flips photos and videos back.
|
||||
class WebDarkMode {
|
||||
static const _css = 'html{filter:invert(1) hue-rotate(180deg)!important;'
|
||||
'background:#fff!important}'
|
||||
'img,picture,video,canvas,iframe,svg image,[style*="background-image"]'
|
||||
'{filter:invert(1) hue-rotate(180deg)!important}';
|
||||
|
||||
static const script = '''
|
||||
(function () {
|
||||
function apply() {
|
||||
if (document.getElementById('tagesbyte-dark')) return;
|
||||
var style = document.createElement('style');
|
||||
style.id = 'tagesbyte-dark';
|
||||
style.textContent = '$_css';
|
||||
(document.head || document.documentElement).appendChild(style);
|
||||
}
|
||||
apply();
|
||||
document.addEventListener('DOMContentLoaded', apply);
|
||||
})();
|
||||
''';
|
||||
}
|
||||
Reference in new issue
Block a user