From a4d9ba11381295380314a1f749c1ae1eeb6773f2 Mon Sep 17 00:00:00 2001 From: Arjun Satarkar
Date: Tue, 4 Jul 2023 21:25:53 +0530 Subject: Add feed modification and deletion --- .gitignore | 3 +- __pycache__/tagrss.cpython-311.pyc | Bin 7735 -> 0 bytes pyrightconfig.json | 3 + requirements.txt | 5 ++ serve.py | 69 ++++++++++++++++---- tagrss.py | 125 ++++++++++++++++++++++++++++++------- views/add_feed.tpl | 1 - views/delete.html | 14 +++++ views/index.tpl | 8 ++- views/manage_feed.tpl | 56 +++++++++++++++++ 10 files changed, 244 insertions(+), 40 deletions(-) delete mode 100644 __pycache__/tagrss.cpython-311.pyc create mode 100644 pyrightconfig.json create mode 100644 views/delete.html create mode 100644 views/manage_feed.tpl diff --git a/.gitignore b/.gitignore index 96e766e..fa064c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +/__pycache__/ /venv/ -/ignore/ \ No newline at end of file +/ignore/ diff --git a/__pycache__/tagrss.cpython-311.pyc b/__pycache__/tagrss.cpython-311.pyc deleted file mode 100644 index d906911..0000000 Binary files a/__pycache__/tagrss.cpython-311.pyc and /dev/null differ diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..6d8dc68 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,3 @@ +{ + "typeCheckingMode": "basic", +} diff --git a/requirements.txt b/requirements.txt index 425cad0..c32839f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ bottle==0.12.25 +certifi==2023.5.7 +charset-normalizer==3.1.0 feedparser==6.0.10 gevent==22.10.2 greenlet==2.0.2 +idna==3.4 +requests==2.31.0 sgmllib3k==1.0.0 +urllib3==2.0.3 zope.event==4.6 zope.interface==6.0 diff --git a/serve.py b/serve.py index 025510a..d6e074b 100755 --- a/serve.py +++ b/serve.py @@ -3,13 +3,12 @@ import gevent.monkey gevent.monkey.patch_all() import bottle -import feedparser import gevent.lock import argparse import os import pathlib -import time +import typing import tagrss @@ -22,7 +21,7 @@ args = parser.parse_args() storage_path: pathlib.Path = pathlib.Path(args.storage_path) tagrss_lock = gevent.lock.RLock() -tagrss_backend = tagrss.TagRss(storage_path=storage_path) +core = tagrss.TagRss(storage_path=storage_path) def parse_space_separated_tags(inp: str) -> list[str]: @@ -44,30 +43,39 @@ def parse_space_separated_tags(inp: str) -> list[str]: tag += c if tag: tags.add(tag) - return tuple(sorted(tags)) + return list(sorted(tags)) + + +def serialise_tags(tags: list[str]) -> str: + result = "" + for i, tag in enumerate(tags): + if i > 0: + result += " " + result += (tag.replace("\\", "\\\\")).replace(" ", "\\ ") + return result @bottle.route("/") def index(): with tagrss_lock: - entries = tagrss_backend.get_entries(limit=100) - return bottle.template("index", entries=entries, tagrss_backend=tagrss_backend) + entries = core.get_entries(limit=100) + return bottle.template("index", entries=entries, core=core) @bottle.get("/add_feed") -def add_feed_ui(): +def add_feed_view(): return bottle.template("add_feed") @bottle.post("/add_feed") def add_feed_effect(): - feed_source = bottle.request.forms.get("feed_source") - tags = parse_space_separated_tags(bottle.request.forms.get("tags")) + feed_source: str = bottle.request.forms.get("feed_source") # type: ignore + tags = parse_space_separated_tags(bottle.request.forms.get("tags")) # type: ignore already_present: bool = False with tagrss_lock: try: - tagrss_backend.add_feed(feed_source=feed_source, tags=tags) + core.add_feed(feed_source=feed_source, tags=tags) except tagrss.FeedAlreadyAddedError: already_present = True # TODO: handle FeedFetchError too @@ -75,13 +83,52 @@ def add_feed_effect(): "add_feed", after_add=True, feed_source=feed_source, - already_present=already_present + already_present=already_present, ) +@bottle.get("/manage_feed") +def manage_feed_view(): + try: + feed_id_raw: str = bottle.request.query["feed"] # type: ignore + feed_id: int = int(feed_id_raw) + except KeyError: + raise bottle.HTTPError(400, "Feed ID not given.") + feed: dict[str, typing.Any] = {} + feed["id"] = feed_id + feed["source"] = core.get_feed_source(feed_id) + feed["title"] = core.get_feed_title(feed_id) + feed["tags"] = core.get_feed_tags(feed_id) + feed["serialised_tags"] = serialise_tags(feed["tags"]) + return bottle.template("manage_feed", feed=feed) + +@bottle.post("/manage_feed") +def manage_feed_effect_update(): + feed_id: int = int(bottle.request.forms["id"]) # type: ignore + feed_source: str = bottle.request.forms["source"] # type: ignore + feed_title: str = bottle.request.forms["title"] # type: ignore + feed_tags: list[str] = parse_space_separated_tags(bottle.request.forms["tags"]) # type: ignore + core.set_feed_source(feed_id, feed_source) + core.set_feed_title(feed_id, feed_title) + core.set_feed_tags(feed_id, feed_tags) + return bottle.redirect(f"/manage_feed?feed={feed_id}") + +@bottle.get("/delete") +def delete_view(): + return bottle.static_file("delete.html", root="views") + + +@bottle.post("/delete") +def delete_effect(): + feed_id: int = int(bottle.request.forms["id"]) # type: ignore + core.delete_feed(feed_id) + return bottle.redirect("/delete") + + @bottle.get("/static/
") def serve_static(path): return bottle.static_file(path, pathlib.Path(os.getcwd(), "static")) bottle.run(host=args.host, port=args.port, server="gevent") +core.close() diff --git a/tagrss.py b/tagrss.py index 7b1d084..c1dfbf5 100644 --- a/tagrss.py +++ b/tagrss.py @@ -18,24 +18,51 @@ class FeedFetchError(Exception): super().__init__(f"Get {feed_source} returned HTTP {status_code}") +class SqliteMissingForeignKeySupportError(Exception): + pass + + class TagRss: def __init__(self, *, storage_path: str | pathlib.Path): self.connection: sqlite3.Connection = sqlite3.connect(storage_path) + with self.connection: self.connection.executescript( """ -CREATE TABLE IF NOT EXISTS feeds(id INTEGER PRIMARY KEY, source TEXT UNIQUE, title TEXT); -CREATE INDEX IF NOT EXISTS feed_source ON feeds(source); +PRAGMA foreign_keys = ON; -CREATE TABLE IF NOT EXISTS feed_tags(feed_id INTEGER, tag TEXT); -CREATE INDEX IF NOT EXISTS feed_tags_feed_id ON feed_tags(feed_id); +CREATE TABLE IF NOT EXISTS + feeds( + id INTEGER PRIMARY KEY, + source TEXT UNIQUE, + title TEXT + ) STRICT; -CREATE TABLE IF NOT EXISTS entries(id INTEGER PRIMARY KEY, feed_id INTEGER, title TEXT, link TEXT, epoch_published INTEGER, epoch_updated INTEGER, epoch_downloaded INTEGER); -CREATE INDEX IF NOT EXISTS entry_epoch_downloaded ON entries(epoch_downloaded); +CREATE TABLE IF NOT EXISTS + feed_tags( + feed_id INTEGER REFERENCES feeds(id) ON DELETE CASCADE, + tag TEXT + ) STRICT; +CREATE INDEX IF NOT EXISTS idx_feed_tags__feed_id__tag ON feed_tags(feed_id, tag); +CREATE INDEX IF NOT EXISTS idx_feed_tags__tag__feed_id ON feed_tags(tag, feed_id); + +CREATE TABLE IF NOT EXISTS + entries( + id INTEGER PRIMARY KEY, + feed_id INTEGER REFERENCES feeds(id) ON DELETE CASCADE, + title TEXT, + link TEXT, + epoch_published INTEGER, + epoch_updated INTEGER, + epoch_downloaded INTEGER + ) STRICT; +CREATE INDEX IF NOT EXISTS idx_entries__epoch_downloaded ON entries(epoch_downloaded); """ ) + if (1,) not in self.connection.execute("PRAGMA foreign_keys;").fetchmany(1): + raise SqliteMissingForeignKeySupportError - def add_feed(self, *, feed_source: str, tags: tuple[str]): + def add_feed(self, feed_source: str, tags: list[str]) -> None: response = requests.get(feed_source) if response.status_code != requests.codes.ok: raise FeedFetchError(feed_source, response.status_code) @@ -44,7 +71,8 @@ CREATE INDEX IF NOT EXISTS entry_epoch_downloaded ON entries(epoch_downloaded); except KeyError: base: str = feed_source parsed = feedparser.parse( - io.BytesIO(bytes(response.text, encoding="utf-8")), response_headers={"Content-Location": base} + io.BytesIO(bytes(response.text, encoding="utf-8")), + response_headers={"Content-Location": base}, ) with self.connection: feed_title: str = parsed.feed.get("title", "") @@ -52,36 +80,37 @@ CREATE INDEX IF NOT EXISTS entry_epoch_downloaded ON entries(epoch_downloaded); self.connection.execute( "INSERT INTO feeds(source, title) VALUES(?, ?);", (feed_source, feed_title), - ) + ) # Note: ensure no more INSERTs between this and the last_insert_rowid() call except sqlite3.IntegrityError: raise FeedAlreadyAddedError feed_id: int = int( - self.connection.execute( - "SELECT id FROM feeds WHERE source = ?;", (feed_source,) - ).fetchone()[0] + self.connection.execute("SELECT last_insert_rowid();").fetchone()[0] ) self.connection.executemany( - f"INSERT INTO feed_tags(feed_id, tag) VALUES({feed_id}, ?);", tuple(((tag,) for tag in tags)) + "INSERT INTO feed_tags(feed_id, tag) VALUES(?, ?);", + ((feed_id, tag) for tag in tags), ) for entry in reversed(parsed.entries): - link: str = entry.get("link", "") + link: str = entry.get("link", None) + title: str = entry.get("title", None) try: epoch_published: typing.Optional[int] = calendar.timegm( entry.get("published_parsed", None) ) - except ValueError: + except TypeError: epoch_published = None try: epoch_updated: typing.Optional[int] = calendar.timegm( entry.get("updated_parsed", None) ) - except ValueError: + except TypeError: epoch_updated = None self.connection.execute( - "INSERT INTO entries(feed_id, title, link, epoch_published, epoch_updated, epoch_downloaded) VALUES(?, ?, ?, ?, ?, ?);", + "INSERT INTO entries(feed_id, title, link, epoch_published, epoch_updated, epoch_downloaded) \ + VALUES(?, ?, ?, ?, ?, ?);", ( feed_id, - feed_title, + title, link, epoch_published, epoch_updated, @@ -89,15 +118,16 @@ CREATE INDEX IF NOT EXISTS entry_epoch_downloaded ON entries(epoch_downloaded); ), ) - def get_entries(self, *, limit: int): + def get_entries(self, *, limit: int) -> list[dict[str, typing.Any]]: with self.connection: - result = self.connection.execute( - "SELECT feed_id, title, link, epoch_published, epoch_updated FROM entries ORDER BY epoch_downloaded DESC LIMIT ?;", + resp = self.connection.execute( + "SELECT feed_id, title, link, epoch_published, epoch_updated FROM entries \ + ORDER BY epoch_downloaded DESC LIMIT ?;", (limit,), ).fetchall() entries = [] - for entry in result: + for entry in resp: entries.append( { "feed_id": entry[0], @@ -108,6 +138,53 @@ CREATE INDEX IF NOT EXISTS entry_epoch_downloaded ON entries(epoch_downloaded); } ) return entries - def get_feed_tags(self, feed_id: int) -> tuple[str]: + + def get_feed_source(self, feed_id: int) -> str: + with self.connection: + return self.connection.execute( + "SELECT source FROM feeds WHERE id = ?;", (feed_id,) + ).fetchone()[0] + + def get_feed_title(self, feed_id: int) -> str: + with self.connection: + return self.connection.execute( + "SELECT title FROM feeds WHERE id = ?;", (feed_id,) + ).fetchone()[0] + + def get_feed_tags(self, feed_id: int) -> list[str]: + with self.connection: + return [ + t[0] + for t in self.connection.execute( + "SELECT tag FROM feed_tags WHERE feed_id = ?;", (feed_id,) + ).fetchall() + ] + + def set_feed_source(self, feed_id: int, feed_source: str): + with self.connection: + self.connection.execute( + "UPDATE feeds SET source = ? WHERE id = ?;", (feed_source, feed_id) + ) + + def set_feed_title(self, feed_id: int, feed_title: str): + with self.connection: + self.connection.execute( + "UPDATE feeds SET title = ? WHERE id = ?;", (feed_title, feed_id) + ) + + def set_feed_tags(self, feed_id: int, feed_tags: list[str]): + with self.connection: + self.connection.execute("DELETE FROM feed_tags WHERE feed_id = ?;", (feed_id,)) + self.connection.executemany( + "INSERT INTO feed_tags(feed_id, tag) VALUES(?, ?);", + ((feed_id, tag) for tag in feed_tags), + ) + + def delete_feed(self, feed_id: int) -> None: + with self.connection: + self.connection.execute("DELETE FROM feeds WHERE id = ?;", (feed_id,)) + + def close(self) -> None: with self.connection: - return tuple((t[0] for t in self.connection.execute("SELECT tag FROM feed_tags WHERE feed_id = ?;", (feed_id,)).fetchall())) \ No newline at end of file + self.connection.execute("PRAGMA optimize;") + self.connection.close() diff --git a/views/add_feed.tpl b/views/add_feed.tpl index 3c54ffb..5fc88fb 100644 --- a/views/add_feed.tpl +++ b/views/add_feed.tpl @@ -23,7 +23,6 @@
🛈
-