Härtung und Sammelpostfach per SSO (#1)
This commit was merged in pull request #1.
This commit is contained in:
9 files changed
+626
-112
No files matched your search
@@ -1,138 +1,207 @@
|
||||
import imaplib
|
||||
"""
|
||||
E-Mail Unsubscribe Link Finder.
|
||||
|
||||
Reads a collection mailbox (forwarded newsletters), lists the unsubscribe links per mail and moves
|
||||
handled mails to the trash. Runs behind the reverse proxy (Caddy forward_auth / Authelia); the
|
||||
mailbox credentials come from the environment.
|
||||
"""
|
||||
|
||||
import email
|
||||
import re
|
||||
import imaplib
|
||||
import logging
|
||||
import os
|
||||
from flask import Flask, render_template, request, redirect, url_for
|
||||
from dotenv import load_dotenv
|
||||
import re
|
||||
from email.header import decode_header, make_header
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from dotenv import load_dotenv
|
||||
from flask import Flask, abort, redirect, render_template, request, url_for
|
||||
|
||||
app = Flask(__name__)
|
||||
load_dotenv() # Laden der Umgebungsvariablen aus der .env-Datei
|
||||
from oauth import MailboxOAuth, OAuthError, xoauth2_string
|
||||
|
||||
# Email-Server-Konfiguration
|
||||
EMAIL_USERNAME = os.environ.get("EMAIL_USERNAME")
|
||||
EMAIL_PASSWORD = os.environ.get("EMAIL_PASSWORD")
|
||||
EMAIL_SERVER = os.environ.get("EMAIL_SERVER")
|
||||
load_dotenv() # optional .env for local runs
|
||||
|
||||
EMAIL_USERNAME = os.environ.get("EMAIL_USERNAME", "")
|
||||
EMAIL_PASSWORD = os.environ.get("EMAIL_PASSWORD", "")
|
||||
EMAIL_SERVER = os.environ.get("EMAIL_SERVER", "")
|
||||
EMAIL_PORT = int(os.environ.get("EMAIL_PORT", "993"))
|
||||
EMAIL_FOLDER = os.environ.get("EMAIL_FOLDER", "INBOX")
|
||||
TRASH_FOLDER = os.environ.get("TRASH_FOLDER", "Trash")
|
||||
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
template_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
app = Flask(__name__, template_folder=template_dir)
|
||||
link_pattern = r"<a href=\"(.*?)\">(.*?)</a>"
|
||||
unsubscribe_pattern = r"(?i)\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b"
|
||||
|
||||
def get_text_from_email(email_message):
|
||||
# OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD
|
||||
OAUTH = MailboxOAuth.from_env()
|
||||
|
||||
UNSUBSCRIBE_PATTERN = re.compile(
|
||||
r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE
|
||||
)
|
||||
URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
|
||||
UID_PATTERN = re.compile(r"^\d+$")
|
||||
|
||||
|
||||
def is_safe_link(url: str) -> bool:
|
||||
"""Only plain web links: mails are untrusted, javascript:/data: links must never become clickable."""
|
||||
return url.lower().startswith(("https://", "http://"))
|
||||
|
||||
|
||||
def get_text_from_email(message) -> str:
|
||||
text = ""
|
||||
for part in email_message.walk():
|
||||
content_type = part.get_content_type()
|
||||
content_disposition = str(part.get("Content-Disposition"))
|
||||
|
||||
if "text" in content_type and "attachment" not in content_disposition:
|
||||
for part in message.walk():
|
||||
disposition = str(part.get("Content-Disposition"))
|
||||
if "text" in part.get_content_type() and "attachment" not in disposition:
|
||||
body = part.get_payload(decode=True)
|
||||
if body:
|
||||
text += body.decode(errors="ignore")
|
||||
return text
|
||||
|
||||
def find_unsubscribe_links(text, html_links):
|
||||
soup = BeautifulSoup(text, 'html.parser')
|
||||
unsubscribe_links = []
|
||||
|
||||
def get_subject(message) -> str:
|
||||
raw = message.get("Subject", "")
|
||||
try:
|
||||
return str(make_header(decode_header(raw)))
|
||||
except Exception:
|
||||
return raw
|
||||
|
||||
|
||||
for link in soup.find_all('a', href=True):
|
||||
if re.search(unsubscribe_pattern, str(link)) and str(link['href']) not in html_links:
|
||||
print(link['href'])
|
||||
test1=str(link['href'])
|
||||
print(test1)
|
||||
if test1 not in unsubscribe_links:
|
||||
print(test1)
|
||||
unsubscribe_links.append(test1)
|
||||
|
||||
def find_unsubscribe_links(text: str) -> list[str]:
|
||||
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs."""
|
||||
links: list[str] = []
|
||||
soup = BeautifulSoup(text, "html.parser")
|
||||
for anchor in soup.find_all("a", href=True):
|
||||
href = str(anchor["href"]).strip()
|
||||
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
|
||||
links.append(href)
|
||||
for url in URL_PATTERN.findall(text):
|
||||
if UNSUBSCRIBE_PATTERN.search(url) and url not in links:
|
||||
links.append(url)
|
||||
return links
|
||||
|
||||
# Zusätzlich nach einfachen URLs suchen und auf das Unsubscribe-Pattern prüfen
|
||||
simple_urls = re.findall(r"(https?://\S+)", text)
|
||||
for url in simple_urls:
|
||||
if re.search(unsubscribe_pattern, url) and url not in html_links:
|
||||
if url not in unsubscribe_links:
|
||||
unsubscribe_links.append(url)
|
||||
|
||||
return unsubscribe_links
|
||||
def connect() -> imaplib.IMAP4_SSL:
|
||||
token = OAUTH.access_token() if OAUTH else None # before connecting: fail fast without a token
|
||||
mail = imaplib.IMAP4_SSL(EMAIL_SERVER, EMAIL_PORT)
|
||||
if token:
|
||||
mail.authenticate("XOAUTH2", lambda _: xoauth2_string(EMAIL_USERNAME, token).encode())
|
||||
else:
|
||||
mail.login(EMAIL_USERNAME, EMAIL_PASSWORD)
|
||||
mail.select(EMAIL_FOLDER)
|
||||
return mail
|
||||
|
||||
def get_subject_from_email(email_message):
|
||||
subject = email_message.get("Subject")
|
||||
return subject
|
||||
|
||||
def move_emails_to_trash(email_ids):
|
||||
def collect_links(mail) -> dict:
|
||||
"""{uid: {"subject": ..., "links": [...]}} for all mails with unsubscribe links (UIDs stay stable)."""
|
||||
_, data = mail.uid("search", None, "ALL")
|
||||
result = {}
|
||||
for uid in data[0].split():
|
||||
_, msg_data = mail.uid("fetch", uid, "(BODY.PEEK[])")
|
||||
if not msg_data or not isinstance(msg_data[0], tuple):
|
||||
continue
|
||||
message = email.message_from_bytes(msg_data[0][1])
|
||||
body = get_text_from_email(message)
|
||||
links = find_unsubscribe_links(body) if body else []
|
||||
if links:
|
||||
result[uid.decode()] = {"subject": get_subject(message), "links": links}
|
||||
return result
|
||||
|
||||
username = EMAIL_USERNAME
|
||||
password = EMAIL_PASSWORD
|
||||
server = EMAIL_SERVER
|
||||
|
||||
mail = imaplib.IMAP4_SSL(server)
|
||||
mail.login(username, password)
|
||||
mail.select("inbox")
|
||||
def move_emails_to_trash(uids: list[str]) -> int:
|
||||
uids = [u for u in uids if UID_PATTERN.match(u)]
|
||||
if not uids:
|
||||
return 0
|
||||
mail = connect()
|
||||
try:
|
||||
for uid in uids:
|
||||
# Mailcow/Dovecot: copy to trash, flag deleted, expunge
|
||||
mail.uid("copy", uid, TRASH_FOLDER)
|
||||
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
mail.expunge()
|
||||
finally:
|
||||
mail.logout()
|
||||
logger.info("Moved %d mail(s) to %s", len(uids), TRASH_FOLDER)
|
||||
return len(uids)
|
||||
|
||||
for email_id in email_ids:
|
||||
# Verschiebe die E-Mail in den Papierkorb (Trash)
|
||||
mail.copy(email_id, "Trash")
|
||||
# Setze die \Deleted-Flagge, um die E-Mail später zu löschen (für Mailcow)
|
||||
mail.store(email_id, "+FLAGS", "\\Deleted")
|
||||
|
||||
# Speichere die Änderungen auf dem IMAP-Server
|
||||
mail.expunge()
|
||||
mail.logout()
|
||||
|
||||
@app.route("/", methods=["GET", "POST"])
|
||||
@app.route("/", methods=["GET"])
|
||||
def index():
|
||||
return render_template("index.html", email_address=EMAIL_USERNAME)
|
||||
return render_template(
|
||||
"index.html",
|
||||
email_address=EMAIL_USERNAME,
|
||||
oauth=OAUTH is not None,
|
||||
connected=OAUTH.connected() if OAUTH else True,
|
||||
error=request.args.get("error", ""),
|
||||
)
|
||||
|
||||
|
||||
@app.route("/connect")
|
||||
def connect_mailbox():
|
||||
"""Connect the collection mailbox: sign in at the SSO as its technical account (private window)."""
|
||||
if not OAUTH:
|
||||
abort(404)
|
||||
return redirect(OAUTH.start())
|
||||
|
||||
|
||||
@app.route("/oidc/callback")
|
||||
def oidc_callback():
|
||||
if not OAUTH:
|
||||
abort(404)
|
||||
if request.args.get("error"):
|
||||
return redirect(url_for("index", error=request.args["error"]))
|
||||
try:
|
||||
OAUTH.finish(request.args.get("state", ""), request.args.get("code", ""))
|
||||
except OAuthError as e:
|
||||
logger.warning("Mailbox connect failed: %s", e)
|
||||
return redirect(url_for("index", error=str(e)))
|
||||
logger.info("Collection mailbox %s connected via OAuth", EMAIL_USERNAME)
|
||||
return redirect(url_for("index"))
|
||||
|
||||
|
||||
@app.route("/disconnect", methods=["POST"])
|
||||
def disconnect_mailbox():
|
||||
if not OAUTH:
|
||||
abort(404)
|
||||
OAUTH.disconnect()
|
||||
return redirect(url_for("index"))
|
||||
|
||||
|
||||
@app.errorhandler(OAuthError)
|
||||
def oauth_error(e):
|
||||
logger.warning("Mailbox access failed: %s", e)
|
||||
return redirect(url_for("index", error="Mailbox not connected or token expired – connect again"))
|
||||
|
||||
|
||||
@app.route("/result")
|
||||
def result():
|
||||
|
||||
username = EMAIL_USERNAME
|
||||
password = EMAIL_PASSWORD
|
||||
server = EMAIL_SERVER
|
||||
|
||||
mail = imaplib.IMAP4_SSL(server)
|
||||
mail.login(username, password)
|
||||
mail.select("inbox")
|
||||
|
||||
_, messages = mail.search(None, "ALL")
|
||||
links_data = {}
|
||||
html_links = set()
|
||||
|
||||
for message_num in messages[0].split():
|
||||
_, msg_data = mail.fetch(message_num, "(RFC822)")
|
||||
msg = email.message_from_bytes(msg_data[0][1])
|
||||
body = get_text_from_email(msg)
|
||||
subject = get_subject_from_email(msg)
|
||||
if body:
|
||||
unsubscribe_links_in_html = find_unsubscribe_links(body, html_links)
|
||||
html_links.update(unsubscribe_links_in_html)
|
||||
unsubscribe_links_in_text = find_unsubscribe_links(body, html_links)
|
||||
all_unsubscribe_links = list(set(unsubscribe_links_in_html + unsubscribe_links_in_text))
|
||||
if all_unsubscribe_links:
|
||||
links_data[message_num.decode()] = {
|
||||
"subject": subject,
|
||||
"links": all_unsubscribe_links
|
||||
}
|
||||
|
||||
mail.logout()
|
||||
|
||||
mail = connect()
|
||||
try:
|
||||
links_data = collect_links(mail)
|
||||
finally:
|
||||
mail.logout()
|
||||
return render_template("result.html", links_data=links_data)
|
||||
|
||||
|
||||
@app.route("/move-to-trash", methods=["POST"])
|
||||
def move_to_trash():
|
||||
if request.method == "POST":
|
||||
email_id = request.form.get("email_id")
|
||||
if email_id:
|
||||
move_emails_to_trash([email_id])
|
||||
email_id = request.form.get("email_id", "")
|
||||
move_emails_to_trash([email_id])
|
||||
return redirect(url_for("result"))
|
||||
|
||||
|
||||
@app.route("/move-all-to-trash", methods=["POST"])
|
||||
def move_all_to_trash():
|
||||
if request.method == "POST":
|
||||
email_ids = request.form.getlist("email_id")
|
||||
if email_ids:
|
||||
move_emails_to_trash(email_ids)
|
||||
move_emails_to_trash(request.form.getlist("email_id"))
|
||||
return redirect(url_for("index"))
|
||||
|
||||
|
||||
@app.route("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="0.0.0.0", port=5000, debug=True)
|
||||
# local development only; the container runs gunicorn (no debugger)
|
||||
app.run(host="127.0.0.1", port=5000, debug=os.environ.get("FLASK_DEBUG") == "1")
|
||||
Reference in new issue
Block a user