""" app.py — QR Carrier PDF generator: single-item and CSV-driven multi-item workflows. Serves the upload UI at /qr/add/ (Apache ProxyPass). Environment variables: QR_WATCH_DIR Root directory that contains job subfolders (default: /var/www/html/qr) QR_BASE_URL Public URL that maps to QR_WATCH_DIR (default: http://fileshare.icastinc.com/qr) PORT Port for the built-in dev server (default: 5001) """ import base64 import csv import html import io import json import os import re import shutil import subprocess import sys import secrets import stat import tempfile import threading import time import uuid import sqlite3 import zipfile from datetime import date as _date_type, datetime as _datetime_type, timedelta as _timedelta_type, timezone as _dt_timezone from pathlib import Path from urllib.parse import quote, unquote from flask import Flask, abort, jsonify, render_template_string, request, send_file from fpdf import FPDF from openpyxl import load_workbook from werkzeug.utils import secure_filename import msal import qrcode as _qrcode_mod from PIL import Image as _pil_image import requests as _http_requests from functools import wraps from collections import OrderedDict as _OrderedDict_type from contextlib import contextmanager as _contextmanager_type from flask import redirect, session, url_for # Allow importing qr_merge from the parent /var/www/html/qr/ directory sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) # Prefer merge_item (DVI-176 spec); fall back to build_carrier_pdf until it lands. try: from qr_merge import merge_item as _qr_merge_item # noqa: E402 def _merge_item(carrier_path: Path, shop_path: Path, output_folder: Path, submittal_path: Path | None = None) -> Path: generated = Path(_qr_merge_item( carrier_sheet_path=str(carrier_path), shop_drawing_path=str(shop_path), output_folder=str(output_folder), submittal_path=str(submittal_path) if submittal_path else None, base_url=_get_qr_base_url(), )) # Overwrite the carrier sheet with the merged output; remove generated file. if generated.resolve() != carrier_path.resolve(): carrier_path.write_bytes(generated.read_bytes()) generated.unlink() return carrier_path except ImportError: # merge_item not yet in qr_merge.py — use build_carrier_pdf directly. from qr_merge import build_carrier_pdf as _build_carrier_pdf # noqa: E402 def _merge_item(carrier_path: Path, shop_path: Path, output_folder: Path, submittal_path: Path | None = None) -> Path: _base_url = _get_qr_base_url() folder_url = f"{_base_url.rstrip('/')}/{quote(output_folder.name)}" shop_url = f"{folder_url}/{quote(shop_path.name)}" submittal_url = f"{folder_url}/{quote(submittal_path.name)}" if submittal_path else "" carrier_bytes = carrier_path.read_bytes() # read before overwriting result_bytes = _build_carrier_pdf(carrier_bytes, submittal_url, shop_url) carrier_path.write_bytes(result_bytes) # overwrite in place return carrier_path # Also keep build_carrier_pdf available for the legacy single-item route. try: from qr_merge import build_carrier_pdf # noqa: E402 except ImportError: build_carrier_pdf = None # type: ignore[assignment] # Notifications core module (DVI-333) — shared Graph send helper + dispatcher. import notifications # noqa: E402 # Bot observability platform (DVI-824) — records runs/steps/logs/artifacts. import bot_diagnostics as _diag_mod app = Flask(__name__) app.config["MAX_CONTENT_LENGTH"] = 500 * 1024 * 1024 # 500 MB (zip files with PDFs) # --------------------------------------------------------------------------- # Azure AD / Microsoft 365 SSO configuration # --------------------------------------------------------------------------- app.config["SECRET_KEY"] = os.environ["FLASK_SECRET_KEY"] _AZ_CLIENT_ID = os.environ["AZURE_CLIENT_ID"] _AZ_SECRET = os.environ["AZURE_CLIENT_SECRET"] _AZ_TENANT = os.environ["AZURE_TENANT_ID"] # Per-tenant group mappings for multi-tenant SSO. # Each tenant lists a base_group (required for any access) and optional # role groups. A role key present here means "check this group ID for # that role"; absence means the role is not available to that tenant. # base_group=None means any user from that tenant is allowed basic access. _AZ_TENANT_CONFIG = { _AZ_TENANT: { "name": "iCast", "base_group": None, "groups": { "wor_admin": "fed82ffb-eb77-41d4-9f50-59c80f1a942f", "togen_admin": "f0e1aca6-fe4f-4f79-9a8b-3ae4bf644db7", "takeoff_workflow": "13a096d3-08d9-40e2-a6ba-affd2f181249", "prod_run": "75cb48a6-8d0d-48e5-a90e-75d5f65453b7", "togen_mts": "5d5e2955-9b1b-4f16-8e8e-94426158c639", "togen_scada": "8da8d55b-f4f7-4801-84e0-ea509eb5b381", "togen_vision": "ea998389-b27d-45e4-b23e-966bb47cb772", }, }, "5040da8d-ce20-4443-a761-20600db23e5e": { "name": "Voyage Technology", "base_group": "f5a4b007-b23a-4db0-9fc7-2e45a08d45b8", "groups": { "togen_admin": "f5a4b007-b23a-4db0-9fc7-2e45a08d45b8", "wor_admin": "f5a4b007-b23a-4db0-9fc7-2e45a08d45b8", }, }, } _AZ_AUTHORITY = "https://login.microsoftonline.com/organizations" _AZ_AUTHORITY_HOME = f"https://login.microsoftonline.com/{_AZ_TENANT}" _AZ_SCOPES = ["User.Read", "GroupMember.Read.All"] _AZ_REDIRECT = "https://togen.icastinc.com/auth/callback" WATCH_DIR = Path(os.environ.get("QR_WATCH_DIR", "/var/www/html/qr")) SHARED_DIR = Path(os.environ.get("QR_SHARED_DIR", "/var/www/html/shared")) WOR_DIR = Path(os.environ.get("WOR_DIR", "/var/www/html/wor")) WOR_MAINTENANCE_DIR = WOR_DIR / "maintenance" WOR_MAINT_INPROGRESS_DIR = WOR_MAINTENANCE_DIR / "in_progress" WOR_MAINT_COMPLETED_DIR = WOR_MAINTENANCE_DIR / "completed" WOR_MAINT_CANCELLED_DIR = WOR_MAINTENANCE_DIR / "cancelled" WOR_MOBILEMACHINE_DIR = WOR_DIR / "mobilemachine" WOR_MM_INPROGRESS_DIR = WOR_MOBILEMACHINE_DIR / "in_progress" WOR_MM_COMPLETED_DIR = WOR_MOBILEMACHINE_DIR / "completed" WOR_MM_CANCELLED_DIR = WOR_MOBILEMACHINE_DIR / "cancelled" WOR_SAFETY_DIR = WOR_DIR / "safety" WOR_SAFETY_INPROGRESS_DIR = WOR_SAFETY_DIR / "in_progress" WOR_SAFETY_COMPLETED_DIR = WOR_SAFETY_DIR / "completed" WOR_SAFETY_CANCELLED_DIR = WOR_SAFETY_DIR / "cancelled" WOR_PREVENTIVE_DIR = WOR_DIR / "preventive" WOR_PREVENTIVE_INPROGRESS_DIR = WOR_PREVENTIVE_DIR / "in_progress" WOR_PREVENTIVE_COMPLETED_DIR = WOR_PREVENTIVE_DIR / "completed" WOR_PREVENTIVE_CANCELLED_DIR = WOR_PREVENTIVE_DIR / "cancelled" WOR_RECURRING_PATH = WOR_DIR / "recurring.json" # Device manuals (DVI-452) live under the WOR tree so they are reachable through # the Document Viewer at the shared URL prefix (e.g. docs.icastinc.com/wor/manuals/). WOR_MANUALS_DIR = WOR_DIR / "manuals" # Equipment/Asset/System -> manual association index (kept beside app state, not web-exposed). WOR_MANUALS_INDEX_FILE = Path(__file__).resolve().parent / "pm_manuals_index.json" # Searchable-manual text index (DVI-491): stored-filename -> OCR text + per-page # offsets. Separate from the association index above, which stays equipment->file. WOR_MANUALS_TEXT_INDEX_FILE = Path(__file__).resolve().parent / "pm_manuals_text_index.json" # Pre-rendered Grid-view thumbnails (DVI-585): small JPEGs cached beside the app # state (not under the web-exposed manuals tree). Keyed by stored filename so the # grid loads a tiny image instead of downloading + rendering each full PDF. WOR_MANUALS_THUMBS_DIR = Path(__file__).resolve().parent / "pm_manuals_thumbs" MANUAL_THUMB_MAX_PX = 480 # longest-edge bound for cached thumbnails ALLOWED_MANUAL_EXTS = {".pdf", ".doc", ".docx", ".png", ".jpg", ".jpeg"} # Legacy aliases kept for backward compat during migration WOR_INPROGRESS_DIR = WOR_MAINT_INPROGRESS_DIR WOR_COMPLETED_DIR = WOR_MAINT_COMPLETED_DIR SUPPORT_DIR = Path(os.environ.get("SUPPORT_DIR", "/var/www/html/support_requests")) FEATURE_DIR = Path(os.environ.get("FEATURE_DIR", "/var/www/html/feature_requests")) PROJECTBUILDER_DIR = Path(os.environ.get("PROJECTBUILDER_DIR", "/var/www/html/projectbuilder")) STOCK_PRODUCTS_DIR = Path(os.environ.get("STOCK_PRODUCTS_DIR", "/var/www/html/stockproducts")) STOCK_DRAWINGS_DIR = STOCK_PRODUCTS_DIR / "drawings" MTS_LABELS_DIR = Path(os.environ.get("MTS_LABELS_DIR", "/var/www/html/mtslabels")) # OCR Tool (DVI-490): job-scoped folders hold the uploaded PDF, the searchable # output, and a status.json. Jobs older than OCR_JOB_TTL_DAYS are pruned. OCR_DIR = Path(os.environ.get("OCR_DIR", "/var/www/html/ocr")) OCR_JOB_TTL_DAYS = 7 BASE_URL = os.environ.get("QR_BASE_URL", "http://fileshare.icastinc.com/qr") AZURE_MAIL_FROM = os.environ.get("AZURE_MAIL_FROM", "") TOOL_VISIBILITY_FILE = Path(__file__).resolve().parent / "tool_visibility.json" TOOL_ORDER_FILE = Path(__file__).resolve().parent / "tool_order.json" # Toolbar/top-bar functions managed from Togen Admin -> Tools -> Toolbars Enabled # (DVI-616). Each has an Admins + Users toggle: "users" => visible to everyone, # "admins" => visible to Togen Admins (iCast + Voyage Technology groups) only. TOOLBAR_VISIBILITY_FILE = Path(__file__).resolve().parent / "toolbar_visibility.json" PUBLIC_VERSION_FILE = Path(__file__).resolve().parent / "public_version.txt" # About text shown in the Help-menu About popup; editable from Togen Admin (DVI-412). ABOUT_CONTENT_FILE = Path(__file__).resolve().parent / "about_content.md" # Header title + tagline shown atop every page; editable from Togen Admin (DVI-555). SITE_TEXT_SETTINGS_FILE = Path(__file__).resolve().parent / "site_text_settings.json" # Per-tool display name/description overrides, editable from Togen Admin (DVI-588). # Shape: {tool_id: {"name": str, "description": str}}. Blank values fall back to # the built-in client defaults, so the file only stores genuine overrides. TOOL_OVERRIDES_FILE = Path(__file__).resolve().parent / "tool_overrides.json" # Desired-state file for the rclone bisync service stop/start toggle (DVI-412). # The actual host service control is wired by the Linux Automation Engineer subtask. SYNC_SERVICE_STATE_FILE = Path(__file__).resolve().parent / "sync_service_state.json" NEWS_FILE = Path(__file__).resolve().parent / "news_posts.json" SUBSCRIBERS_FILE = Path(__file__).resolve().parent / "news_subscribers.json" SHARED_URL_PREFIX_FILE = Path(__file__).resolve().parent / "shared_url_prefix.json" USER_SETTINGS_DIR = Path(__file__).resolve().parent / "user_settings" # File Browser folder-watcher snapshots (DVI-471). Keyed by watcher id; written # only by the background watcher loop so it never races user-settings writes. FB_WATCHER_STATE_FILE = Path(__file__).resolve().parent / "fb_watcher_state.json" SCADA_CONFIG_FILE = Path(__file__).resolve().parent / "scada_config.json" SCADA_REPORT_SCHEDULES_FILE = Path(__file__).resolve().parent / "scada_report_schedules.json" SCADA_ENFORCED_RECIPIENTS_FILE = Path(__file__).resolve().parent / "scada_enforced_recipients.json" SCADA_EXTERNAL_RECIPIENTS_FILE = Path(__file__).resolve().parent / "scada_external_recipients.json" # Produced XLSX reports persisted by scada_report_scheduler.py. Must match that # script's SCADA_REPORTS_DIR (defaults to /reports) so the File Browser # SCADA tab lists the same files the scheduler writes. SCADA_REPORTS_DIR = Path( os.environ.get("SCADA_REPORTS_DIR", str(Path(__file__).resolve().parent / "reports"))) # Vision tool (DVI-399) state files. VISION_COLLECTORS_FILE = Path(__file__).resolve().parent / "vision_collectors.json" VISION_SETTINGS_FILE = Path(__file__).resolve().parent / "vision_settings.json" VISION_AGENTS_FILE = Path(__file__).resolve().parent / "vision_agents.json" # DVI-682: friendly display-name overrides keyed by machine_id. Lets admins # present a human name instead of the raw hostname on the Collector tiles. VISION_AGENT_NAMES_FILE = Path(__file__).resolve().parent / "vision_agent_names.json" # SQLite DB for Vision auth & check-in history (DVI-408). VISION_DB_PATH = Path(__file__).resolve().parent / "vision.db" # A collector is considered online if it checked in within this many seconds. VISION_STALE_SECONDS = 180 # Phase 3 (DVI-427): per-machine capture policy persisted server-side and pushed # to agents via the existing check-in response. VISION_POLICY_FILE = Path(__file__).resolve().parent / "vision_policy.json" # Captured footage store (D3). The Windows Engineer's capture+send task rsyncs # frames here under a separate backlogged task; Togen only reads/serves them. # Path is configurable (never hardcode the rsync host) and defaults next to the # app. Layout: //.jpg (+ .thumb.jpg). VISION_FOOTAGE_DIR = Path( os.environ.get("VISION_FOOTAGE_DIR") or (Path(__file__).resolve().parent / "vision_footage") ) # Footage backend selector (DVI-743). All footage access is routed through a # FootageStore so the local-disk mirror can be swapped for an on-demand remote # backend without touching the serving endpoints or the auth/quarantine # contract. "local" (default) serves the rsync mirror; "remote" (DVI-748 Phase 2) # streams on demand from the storage server over SFTP, no full local copy. VISION_FOOTAGE_BACKEND = (os.environ.get("VISION_FOOTAGE_BACKEND") or "local").strip().lower() # Remote backend (DVI-748): SFTP source + bounded on-disk LRU hot-cache. Host / # port / user reuse the rsync-server config below; the SSH key here is the # read-only footage-pull identity (Phase-3 transport task deploys it on the app # host). Cache caps keep the hot-set bounded — no full mirror (D3=ii). VISION_FOOTAGE_SSH_KEY = ( os.environ.get("VISION_FOOTAGE_SSH_KEY") or str(Path.home() / ".ssh" / "id_togen_rsync_pull") ) VISION_FOOTAGE_REMOTE_BASE = (os.environ.get("VISION_FOOTAGE_REMOTE_BASE") or "footage").strip() VISION_FOOTAGE_CACHE_DIR = Path( os.environ.get("VISION_FOOTAGE_CACHE_DIR") or (Path(__file__).resolve().parent / "vision_footage_cache") ) VISION_FOOTAGE_CACHE_MAX_BYTES = int( os.environ.get("VISION_FOOTAGE_CACHE_MAX_BYTES") or str(512 * 1024 * 1024) ) VISION_FOOTAGE_CACHE_TTL = int(os.environ.get("VISION_FOOTAGE_CACHE_TTL") or "1800") # Pooled SSH connections + a short listing-cache TTL avoid a per-request SFTP # handshake / directory re-scan. VISION_FOOTAGE_SSH_POOL_SIZE = int(os.environ.get("VISION_FOOTAGE_SSH_POOL_SIZE") or "4") VISION_FOOTAGE_LISTING_TTL = int(os.environ.get("VISION_FOOTAGE_LISTING_TTL") or "10") # Frozen agent distribution (DVI-436). The Windows Engineer's packaging task drops # the bundled agent ZIP (frozen exe + installer) here; Togen only serves it for the # Enroll tab's "download installer" action. Configurable; defaults next to the app. VISION_AGENT_DIST_DIR = Path( os.environ.get("VISION_AGENT_DIST_DIR") or (Path(__file__).resolve().parent / "agent_dist") ) VISION_AGENT_ZIP_NAME = os.environ.get("VISION_AGENT_ZIP_NAME") or "togen-vision-agent.zip" # Rsync footage server — connection details used by the server to write approved # agents' SSH public keys into aiv@:~/.ssh/authorized_keys (DVI-726). # VISION_RSYNC_MGMT_KEY must be a private key pre-authorized on the rsync server. _VISION_RSYNC_HOST = os.environ.get("VISION_RSYNC_HOST", "165.188.108.55") _VISION_RSYNC_PORT = int(os.environ.get("VISION_RSYNC_PORT", "4423")) _VISION_RSYNC_USER = os.environ.get("VISION_RSYNC_USER", "aiv") _VISION_RSYNC_MGMT_KEY = (os.environ.get("VISION_RSYNC_MGMT_KEY", "") or "").strip() # Print system (DVI-667 / DVI-657 Print System P1). Mirrors the Vision agent # machinery: a Windows Print Agent enrolls, polls /print/checkin, reports its # discovered printers, and drains a FIFO command queue. PRINT_DB_PATH = Path(__file__).resolve().parent / "print.db" # History capture (DVI-789). Stores cross-tool audit events in one lightweight DB. HISTORY_DB_PATH = Path(__file__).resolve().parent / "history.db" HISTORY_SETTINGS_FILE = Path(__file__).resolve().parent / "history_settings.json" # A print agent is considered online if it checked in within this many seconds. PRINT_STALE_SECONDS = 60 # Frozen print-agent distribution ZIP (built by build_print_agent.ps1 on Windows). PRINT_AGENT_DIST_DIR = Path( os.environ.get("PRINT_AGENT_DIST_DIR") or (Path(__file__).resolve().parent / "agent_dist") ) PRINT_AGENT_ZIP_NAME = os.environ.get("PRINT_AGENT_ZIP_NAME") or "togen-print-agent.zip" # Branding framework (DVI-357). Corporate asset files are dropped in by the # Graphic Designer (DVI-356); their absence falls back to the Togen defaults. BRANDING_SETTINGS_FILE = Path(__file__).resolve().parent / "branding_settings.json" # Printing defaults (DVI-548). Printer options are supplied by Print agents # (not yet wired); only the chosen default printer names persist here. PRINTING_SETTINGS_FILE = Path(__file__).resolve().parent / "printing_settings.json" WINSTON_SETTINGS_FILE = Path(__file__).resolve().parent / "winston_settings.json" WINSTON_SCHEDULES_FILE = Path(__file__).resolve().parent / "winston_schedules.json" WINSTON_RUN_RECORDS_FILE = Path(__file__).resolve().parent / "winston_run_records.json" CORPORATE_LOGO_FILE = Path(__file__).resolve().parent / "corporate_logo.svg" CORPORATE_BG_LIGHT_FILE = Path(__file__).resolve().parent / "corporate_bg_light.svg" CORPORATE_BG_DARK_FILE = Path(__file__).resolve().parent / "corporate_bg_dark.svg" DEFAULT_BRANDING_SETTINGS = {"logo_mode": "togen", "bg_mode": "togen_default", "bg_opacity_light": 100, "bg_opacity_dark": 100, "tile_columns": 3, "tile_side_padding": 0} BRANDING_LOGO_MODES = ("togen", "corporate") BRANDING_BG_MODES = ("togen_default", "flat", "corporate") # Per-user custom background (DVI-518). A user may upload their own image to use # as their Togen interface background instead of the company default/corporate # options. Stored per-user under CUSTOM_BG_DIR keyed by the same safe-email slug # used for settings files; the chosen extension is mirrored in user settings as # "custom_bg_ext" so the client knows the asset exists. "custom" is a per-user # mode only — it is never a company default, so it is excluded from the admin # BRANDING_BG_MODES set above. CUSTOM_BG_DIR = Path(__file__).resolve().parent / "custom_backgrounds" CUSTOM_BG_EXT_MIMES = { "png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "webp": "image/webp", "gif": "image/gif", "svg": "image/svg+xml", } CUSTOM_BG_MAX_BYTES = 8 * 1024 * 1024 # 8 MB USER_BG_MODES = BRANDING_BG_MODES + ("custom",) # Home tile grid layout (DVI-512). "tile_columns" is the number of columns the # home grid is laid out in (D1 — columns, not per-tile pixels), an int in # [3, 9] or the string "unlimited" (auto-pack as many as fit). "tile_view" # (per-user only) selects the grid or compact list presentation (D4). TILE_COLUMNS_MIN = 3 TILE_COLUMNS_MAX = 9 TILE_VIEW_MODES = ("grid", "list") def _coerce_tile_columns(value): """Return "unlimited" or an int in [3, 9] for a valid input, else None.""" if value == "unlimited": return "unlimited" try: n = int(value) except (TypeError, ValueError): return None if TILE_COLUMNS_MIN <= n <= TILE_COLUMNS_MAX: return n return None # Home tile grid side padding (DVI-515). "tile_side_padding" is a percentage of # the available width applied as equal left/right padding on the home grid so # tiles get breathing room on the sides. Admin sets the default; users may # override (blank/null = inherit). Clamped to [0, 40] so content never vanishes. TILE_SIDE_PADDING_MIN = 0 TILE_SIDE_PADDING_MAX = 40 def _coerce_tile_side_padding(value): """Return an int in [0, 40] for a valid percentage input, else None.""" try: n = int(value) except (TypeError, ValueError): return None if TILE_SIDE_PADDING_MIN <= n <= TILE_SIDE_PADDING_MAX: return n return None # Tile/pane transparency (DVI-364). Only takes effect over the corporate # background; clamped to a 40% floor so panes never become unreadable. BRANDING_OPACITY_MIN = 40 BRANDING_OPACITY_MAX = 100 def _coerce_bg_opacity(value): """Return an int in [40, 100] for a valid opacity input, else None.""" try: n = int(value) except (TypeError, ValueError): return None if BRANDING_OPACITY_MIN <= n <= BRANDING_OPACITY_MAX: return n return None SHARED_URL_PREFIX_OPTIONS = [ {"value": "https://docs.icastinc.com", "label": "docs.icastinc.com/shared (HTTPS)"}, {"value": "https://fileshare.icastinc.com", "label": "fileshare.icastinc.com/shared (HTTPS)"}, {"value": "http://fileshare.icastinc.com", "label": "fileshare.icastinc.com/shared (HTTP)"}, ] DEFAULT_SHARED_URL_PREFIX = "https://docs.icastinc.com" _ALL_TOOL_IDS = [ "project-builder", "takeoff-bridge", "prod-run-merge", "work-order", "wor-admin", "sync-admin", "file-browser", "simple-qr", "single-carrier-sheet", "mts-label", "scada-view", "vision", "equipment-manuals", "calendar", "ocr-tool", "email", "auditor", "repair-parts-manager", "bots", ] # Tools that default to hidden in the admin Tools-Enabled list. They remain # always visible to Togen Admins but stay off for everyone else until an admin # enables them (DVI-569). Access is still enforced separately by role gating. # repair-parts-manager (DVI-596) defaults off so it stays hidden for Work Order # Admins until an admin enables it; Togen Admins always see it. # bots (DVI-803) defaults off so it stays hidden for non-admins until an admin # enables it; Togen Admins always see it. _TOOL_DEFAULT_DISABLED = {"auditor", "repair-parts-manager", "bots"} # Toolbar/top-bar functions (DVI-616). Listed in the Toolbars Enabled admin pane. _TOOLBAR_IDS = ["search", "assistant", "calendar"] _TOOLBAR_LABELS = {"search": "Search", "assistant": "Assistant", "calendar": "Calendar"} # Work Order types managed as subitems of the "work-order" tool (DVI-349). _WOR_TYPE_IDS = ["maintenance", "mobilemachine", "safety", "preventive", "ithelpdesk"] WOR_TYPE_VISIBILITY_FILE = Path(__file__).resolve().parent / "wor_type_visibility.json" WOR_TYPE_ORDER_FILE = Path(__file__).resolve().parent / "wor_type_order.json" WOR_PRIMARY_FILE = Path(__file__).resolve().parent / "wor_primary.json" WOR_POLICY_FILE = Path(__file__).resolve().parent / "wor_policy.json" # Repair Parts Manager (DVI-596). Editable Part Number -> Part Name catalog used # to autofill the Part Name when a previously-seen Part Number is entered. The # parts themselves stay on each Work Order sidecar (Option A: WO is the source of # truth); this file is only the known-parts lookup. RPM_KNOWN_PARTS_FILE = Path(__file__).resolve().parent / "rpm_known_parts.json" # RPM tool policy/settings (DVI-607). Currently a single policy toggle. RPM_SETTINGS_FILE = Path(__file__).resolve().parent / "rpm_settings.json" # RPM approval permission + auto-approval policy (DVI-614). Who may approve parts # and which work-order creators get their parts auto-approved on submission. RPM_APPROVAL_FILE = Path(__file__).resolve().parent / "rpm_approval.json" # RPM live carrier-tracking config (DVI-612). Provider + API key + poll cadence # for the background poller that populates tracking_status/delivered_at on parts. # Kept separate from rpm_settings.json because it holds a credential (string), not # the bool-only policy toggles. Disabled-until-configured: no key => poller no-ops. RPM_TRACKING_FILE = Path(__file__).resolve().parent / "rpm_tracking.json" # Work Order types that record/track repair parts (RPM tabs + WO Admin Parts icon). RPM_PART_WOR_TYPES = ("mobilemachine", "preventive") def _load_toolbar_visibility(): """Return {toolbar_id: {"admins": bool, "users": bool}} for top-bar functions. Defaults to both enabled so existing behavior (Search/Assistant/Calendar shown to everyone) is preserved until an admin changes it (DVI-616). """ defaults = {tid: {"admins": True, "users": True} for tid in _TOOLBAR_IDS} if TOOLBAR_VISIBILITY_FILE.is_file(): try: saved = json.loads(TOOLBAR_VISIBILITY_FILE.read_text()) for tid in _TOOLBAR_IDS: row = saved.get(tid) if isinstance(row, dict): defaults[tid] = { "admins": bool(row.get("admins", True)), "users": bool(row.get("users", True)), } except (json.JSONDecodeError, OSError): pass return defaults def _toolbar_enabled_for(toolbar_id, is_admin, vis=None): """Whether a top-bar function is visible/usable for the current user. "users" enabled => visible to everyone; otherwise visible only to Togen Admins when "admins" is enabled. """ vis = vis or _load_toolbar_visibility() row = vis.get(toolbar_id, {"admins": True, "users": True}) if row.get("users"): return True return bool(is_admin and row.get("admins")) def _load_tool_visibility(): """Return dict of {tool_id: bool} for all manageable tools. Tools in ``_TOOL_DEFAULT_DISABLED`` start hidden so they stay invisible to non-admins until an admin explicitly enables them (DVI-569: Auditor). """ defaults = {tid: tid not in _TOOL_DEFAULT_DISABLED for tid in _ALL_TOOL_IDS} if TOOL_VISIBILITY_FILE.is_file(): try: saved = json.loads(TOOL_VISIBILITY_FILE.read_text()) for tid in _ALL_TOOL_IDS: if tid in saved: defaults[tid] = bool(saved[tid]) except (json.JSONDecodeError, OSError): pass return defaults def _load_tool_order(): """Return deduplicated list of tool IDs in display order.""" if TOOL_ORDER_FILE.is_file(): try: saved = json.loads(TOOL_ORDER_FILE.read_text()) if isinstance(saved, list): seen, deduped = set(), [] for tid in saved: if tid not in seen: seen.add(tid) deduped.append(tid) for tid in _ALL_TOOL_IDS: if tid not in seen: seen.add(tid) deduped.append(tid) return deduped except (json.JSONDecodeError, OSError): pass return list(_ALL_TOOL_IDS) def _load_tool_overrides(): """Return {tool_id: {"name": str, "description": str}} admin overrides for a tool's display name/description (DVI-588). Only known tool IDs are kept and blank values are dropped, so the client can safely fall back to its built-in defaults for anything not present here. """ out = {} if TOOL_OVERRIDES_FILE.is_file(): try: saved = json.loads(TOOL_OVERRIDES_FILE.read_text()) if isinstance(saved, dict): for tid, ov in saved.items(): if tid not in _ALL_TOOL_IDS or not isinstance(ov, dict): continue entry = {} name = str(ov.get("name", "")).strip() desc = str(ov.get("description", "")).strip() if name: entry["name"] = name if desc: entry["description"] = desc if entry: out[tid] = entry except (json.JSONDecodeError, OSError): pass return out def _load_wor_type_visibility(): """Return dict of {wor_type: bool} for each Work Order type.""" defaults = {tid: True for tid in _WOR_TYPE_IDS} if WOR_TYPE_VISIBILITY_FILE.is_file(): try: saved = json.loads(WOR_TYPE_VISIBILITY_FILE.read_text()) for tid in _WOR_TYPE_IDS: if tid in saved: defaults[tid] = bool(saved[tid]) except (json.JSONDecodeError, OSError): pass return defaults def _load_wor_type_order(): """Return deduplicated list of Work Order type IDs in display order.""" if WOR_TYPE_ORDER_FILE.is_file(): try: saved = json.loads(WOR_TYPE_ORDER_FILE.read_text()) if isinstance(saved, list): seen, deduped = set(), [] for tid in saved: if tid in _WOR_TYPE_IDS and tid not in seen: seen.add(tid) deduped.append(tid) for tid in _WOR_TYPE_IDS: if tid not in seen: seen.add(tid) deduped.append(tid) return deduped except (json.JSONDecodeError, OSError): pass return list(_WOR_TYPE_IDS) def _user_can_access_pm(user: dict) -> bool: """Preventative Maintenance is role-restricted (DVI-462). Only Work Order Admins and Togen Admins may see the PM tile or submit a PM work order. The ``togen_admin`` role already maps to both the iCast and the Voyage Technology Togen-Admin Azure groups (see ``_AZ_TENANT_CONFIG``), so those groups are covered by this single check. """ user = user or {} return bool(user.get("wor_admin") or user.get("togen_admin")) def _wor_type_enabled_for(wor_type: str, user: dict) -> bool: """Whether *wor_type* is usable by *user*. Togen Admins bypass the gate.""" # Preventative Maintenance is restricted to specific roles regardless of the # admin visibility toggle (DVI-462). if wor_type == "preventive": return _user_can_access_pm(user) if user.get("togen_admin"): return True return _load_wor_type_visibility().get(wor_type, True) def _user_settings_path(email: str) -> Path: """Return the JSON settings file path for a given user email.""" safe = re.sub(r'[^a-zA-Z0-9@._-]', '_', email.lower()) return USER_SETTINGS_DIR / f"{safe}.json" def _load_user_settings(email: str) -> dict: """Return per-user settings dict (tool_visibility, tool_order, theme).""" fp = _user_settings_path(email) if fp.is_file(): try: return json.loads(fp.read_text()) except (json.JSONDecodeError, OSError): pass return {} def _save_user_settings(email: str, settings: dict): """Persist per-user settings to disk.""" USER_SETTINGS_DIR.mkdir(parents=True, exist_ok=True) fp = _user_settings_path(email) fp.write_text(json.dumps(settings, indent=2)) fp.chmod(stat.S_IRUSR | stat.S_IWUSR) def _custom_bg_path(email: str, ext: str) -> Path: """Return the on-disk path for a user's custom background with the given ext.""" safe = re.sub(r'[^a-zA-Z0-9@._-]', '_', email.lower()) return CUSTOM_BG_DIR / f"{safe}.{ext}" def _find_custom_bg(email: str): """Return the user's existing custom-background file (any ext), or None.""" safe = re.sub(r'[^a-zA-Z0-9@._-]', '_', email.lower()) for ext in CUSTOM_BG_EXT_MIMES: fp = CUSTOM_BG_DIR / f"{safe}.{ext}" if fp.is_file(): return fp return None # --------------------------------------------------------------------------- # Notification preference grid (DVI-335) # # The UI renders a friendly 3x5 grid (work-order type x lifecycle column). The # core module (notifications.py) resolves recipients by reading # settings["wo_notifications"][wor_type][], so we translate the # friendly columns into the core's event constants on save and back on load. # "status_change" covers both the in-progress and cancelled transitions; # "completed" has its own column. All entries default off. # --------------------------------------------------------------------------- _NOTIFY_GRID_TYPES = ("maintenance", "mobilemachine", "safety", "preventive") _NOTIFY_GRID_COLUMNS = ("new", "status_change", "completed", "due_soon", "overdue") _NOTIFY_COLUMN_EVENTS = { "new": (notifications.EVENT_WO_CREATED,), "status_change": (notifications.EVENT_WO_STATUS_IN_PROGRESS, notifications.EVENT_WO_STATUS_CANCELLED), "completed": (notifications.EVENT_WO_STATUS_COMPLETED,), "due_soon": (notifications.EVENT_WO_RECURRING_DUE_SOON,), "overdue": (notifications.EVENT_WO_RECURRING_OVERDUE,), } def _notification_grid_from_settings(settings: dict) -> dict: """Derive the friendly 3x5 grid (all-off default) from stored prefs.""" stored = settings.get("wo_notifications") or {} grid = {} for wt in _NOTIFY_GRID_TYPES: type_prefs = stored.get(wt) or {} grid[wt] = { col: any(bool(type_prefs.get(ev)) for ev in _NOTIFY_COLUMN_EVENTS[col]) for col in _NOTIFY_GRID_COLUMNS } return grid def _notification_grid_to_storage(grid: dict) -> dict: """Translate the friendly grid into the core module's event-keyed shape. Only enabled events are stored; all-off types are omitted entirely so the absence of an entry reads as "off" in the core resolver. """ out = {} for wt in _NOTIFY_GRID_TYPES: col_vals = grid.get(wt) or {} events = {} for col in _NOTIFY_GRID_COLUMNS: if col_vals.get(col): for ev in _NOTIFY_COLUMN_EVENTS[col]: events[ev] = True if events: out[wt] = events return out def _user_settings_with_grid(email: str) -> dict: """Load per-user settings plus the derived friendly notifications grid.""" settings = dict(_load_user_settings(email)) settings["notifications"] = _notification_grid_from_settings(settings) return settings # --------------------------------------------------------------------------- # Primary recipients (DVI-405) # # Company-level (not per-user) lists of email addresses, keyed by WO type. A # Primary recipient receives EVERY notification for that WO type, with all # User Settings grid checkboxes enforced on and locked. notifications.py reads # the same wor_primary.json file as an independent recipient source. # --------------------------------------------------------------------------- def _load_wor_primary() -> dict: """Return Primary email lists keyed by WO type, normalised to all types.""" data = {} if WOR_PRIMARY_FILE.is_file(): try: raw = json.loads(WOR_PRIMARY_FILE.read_text()) if isinstance(raw, dict): data = raw except (json.JSONDecodeError, OSError): pass out = {} for wt in _NOTIFY_GRID_TYPES: emails = data.get(wt) out[wt] = [e for e in emails if isinstance(e, str)] if isinstance(emails, list) else [] return out def _save_wor_primary(data: dict) -> None: """Persist Primary lists, de-duplicated and lowercased per WO type.""" clean = {} for wt in _NOTIFY_GRID_TYPES: seen, ordered = set(), [] for e in (data.get(wt) or []): if not isinstance(e, str): continue norm = e.strip().lower() if norm and norm not in seen: seen.add(norm) ordered.append(norm) clean[wt] = ordered WOR_PRIMARY_FILE.write_text(json.dumps(clean, indent=2)) def _wor_primary_enforced_for(email: str) -> list: """WO types for which this email is a Primary recipient (grid enforced on).""" if not email: return [] norm = email.strip().lower() primary = _load_wor_primary() return [wt for wt in _NOTIFY_GRID_TYPES if norm in primary.get(wt, [])] # --------------------------------------------------------------------------- # Policy-enforced notifications (DVI-597) # # Company-level (not per-user) WO type x event grid. When a cell is enabled, the # REQUESTING USER of every work order of that type is notified for that event as # a matter of policy, regardless of their per-WO opt-in. notifications.py reads # the same wor_policy.json file as an independent recipient source. Stored in the # core's event-keyed shape (same as the per-user grid) so the resolver can read # it directly without translation. # --------------------------------------------------------------------------- def _load_wor_policy_grid() -> dict: """Return the friendly Policy grid (all-off default) from wor_policy.json.""" stored = {} if WOR_POLICY_FILE.is_file(): try: raw = json.loads(WOR_POLICY_FILE.read_text()) if isinstance(raw, dict): stored = raw except (json.JSONDecodeError, OSError): pass return _notification_grid_from_settings({"wo_notifications": stored}) def _save_wor_policy_grid(grid: dict) -> None: """Persist the friendly Policy grid in the core's event-keyed shape.""" WOR_POLICY_FILE.write_text( json.dumps(_notification_grid_to_storage(grid), indent=2)) def _load_public_version(): """Return the public version string.""" if PUBLIC_VERSION_FILE.is_file(): return PUBLIC_VERSION_FILE.read_text().strip() return "Unknown" # Default About prose (Markdown) — mirrors the original hard-coded popup text. DEFAULT_ABOUT_MARKDOWN = ( "Togen is iCast's integrated tools platform for bridging material quantity " "estimation to production. From take-off data to QR-coded labels, work orders, " "SCADA monitoring, document sync and preview, and daily production runs — " "Togen pulls it forward.\n\n" "Why “togen”: Togen is the past participle of the Old English verb " "“tēon,” which means to draw or pull.\n\n" "The term ‘togen’ in Old English literature is often associated with the " "concept of ‘fate’ or ‘destiny’, reflecting the beliefs of the " "Anglo-Saxon culture regarding the inevitability of certain events.\n\n" "Or its short for TOol GENerator. Built as a platform to build and access tools, " "chain them into workflows, and impact operations." ) def _load_about_content(): """Return the editable About popup text (Markdown).""" if ABOUT_CONTENT_FILE.is_file(): try: text = ABOUT_CONTENT_FILE.read_text() if text.strip(): return text except OSError: pass return DEFAULT_ABOUT_MARKDOWN def _save_about_content(text): ABOUT_CONTENT_FILE.write_text(text) # Site title + tagline shown in the header on every page; editable from Togen # Admin → General (DVI-555). Falls back to the original hard-coded values. DEFAULT_SITE_TITLE = "togen" DEFAULT_SITE_TAGLINE = "Pull it Forward" def _load_site_text(): """Return the editable header title/tagline, falling back to defaults.""" title, tagline = DEFAULT_SITE_TITLE, DEFAULT_SITE_TAGLINE if SITE_TEXT_SETTINGS_FILE.is_file(): try: saved = json.loads(SITE_TEXT_SETTINGS_FILE.read_text()) if isinstance(saved, dict): if isinstance(saved.get("title"), str) and saved["title"].strip(): title = saved["title"].strip() if isinstance(saved.get("tagline"), str) and saved["tagline"].strip(): tagline = saved["tagline"].strip() except (json.JSONDecodeError, OSError): pass return {"title": title, "tagline": tagline} def _save_site_text(title, tagline): SITE_TEXT_SETTINGS_FILE.write_text( json.dumps({"title": title, "tagline": tagline}, indent=2) ) _BISYNC_UNIT = "rclone-bisync.service" def _load_sync_service_running(): """Return the real run-state of the rclone bisync service. Queries systemctl first; falls back to the persisted desired-state file when systemctl is unavailable (dev environment, insufficient permissions, etc.). """ try: result = subprocess.run( ["systemctl", "is-active", _BISYNC_UNIT], capture_output=True, text=True, timeout=5, ) # "active" → rc 0; "inactive"/"failed"/unknown → non-zero return result.returncode == 0 except Exception: pass # Fallback: persisted desired state if SYNC_SERVICE_STATE_FILE.is_file(): try: data = json.loads(SYNC_SERVICE_STATE_FILE.read_text()) if isinstance(data, dict) and "running" in data: return bool(data["running"]) except (json.JSONDecodeError, OSError): pass return True def _save_sync_service_running(running): SYNC_SERVICE_STATE_FILE.write_text(json.dumps({"running": bool(running)}, indent=2)) def _load_news_posts(): """Return list of news posts, newest first.""" if NEWS_FILE.is_file(): try: posts = json.loads(NEWS_FILE.read_text()) if isinstance(posts, list): posts.sort(key=lambda p: p.get("date", ""), reverse=True) return posts except (json.JSONDecodeError, OSError): pass return [] def _save_news_posts(posts): NEWS_FILE.write_text(json.dumps(posts, indent=2)) def _load_subscribers(): """Return list of subscriber email addresses.""" if SUBSCRIBERS_FILE.is_file(): try: subs = json.loads(SUBSCRIBERS_FILE.read_text()) if isinstance(subs, list): return subs except (json.JSONDecodeError, OSError): pass return [] def _save_subscribers(subs): SUBSCRIBERS_FILE.write_text(json.dumps(subs, indent=2)) def _load_shared_url_prefix(): """Return the configured shared URL prefix string.""" if SHARED_URL_PREFIX_FILE.is_file(): try: data = json.loads(SHARED_URL_PREFIX_FILE.read_text()) if isinstance(data, dict) and data.get("prefix"): return data["prefix"] except (json.JSONDecodeError, OSError): pass return DEFAULT_SHARED_URL_PREFIX def _save_shared_url_prefix(prefix): SHARED_URL_PREFIX_FILE.write_text(json.dumps({"prefix": prefix}, indent=2)) # --- Device manuals (DVI-452) ------------------------------------------------- def _normalize_equipment(name: str) -> str: """Case/whitespace-insensitive key for matching Equipment/Asset/System names.""" return " ".join((name or "").strip().lower().split()) def _manual_viewer_url(stored_filename: str) -> str: """Document Viewer URL for a stored manual file.""" prefix = _load_shared_url_prefix() return f"{prefix}/wor/manuals/{stored_filename}" def _load_manuals_index() -> dict: """Return the {normalized-equipment: manual-entry} association map.""" if WOR_MANUALS_INDEX_FILE.is_file(): try: data = json.loads(WOR_MANUALS_INDEX_FILE.read_text()) if isinstance(data, dict): return data except (json.JSONDecodeError, OSError): pass return {} def _save_manuals_index(index: dict) -> None: WOR_MANUALS_INDEX_FILE.write_text(json.dumps(index, indent=2)) def _manuals_index_set(equipment: str, stored_filename: str, original_name: str, user: dict) -> None: """Associate a manual with an equipment name so future WORs can reuse it.""" key = _normalize_equipment(equipment) if not key: return index = _load_manuals_index() index[key] = { "filename": stored_filename, "original_name": original_name, "equipment": (equipment or "").strip(), "url": _manual_viewer_url(stored_filename), "updated_at": _datetime_type.now().strftime("%Y-%m-%dT%H:%M:%S"), "updated_by": (user or {}).get("name", ""), } _save_manuals_index(index) def _list_all_manuals() -> list: """Return every manual in the association index whose file still exists. Covers both standalone uploads (Equipment Manuals tool) and manuals attached to Preventative Maintenance work orders, since both paths write the same index. """ items = [] for entry in _load_manuals_index().values(): fn = entry.get("filename") if not fn or not (WOR_MANUALS_DIR / fn).is_file(): continue items.append({ "filename": fn, "original_name": entry.get("original_name", fn), "equipment": entry.get("equipment", ""), "url": _manual_viewer_url(fn), "updated_at": entry.get("updated_at", ""), "updated_by": entry.get("updated_by", ""), }) items.sort(key=lambda x: x.get("updated_at", ""), reverse=True) return items def _manuals_index_lookup(equipment: str) -> dict | None: """Return the manual entry previously associated with an equipment name, if any.""" key = _normalize_equipment(equipment) if not key: return None entry = _load_manuals_index().get(key) if not entry or not entry.get("filename"): return None # Only suggest manuals whose file still exists on disk. if not (WOR_MANUALS_DIR / entry["filename"]).is_file(): return None # Refresh URL in case the shared prefix changed since it was stored. entry = dict(entry) entry["url"] = _manual_viewer_url(entry["filename"]) return entry def _collect_pm_eas_options() -> list[str]: """Distinct Equipment/Asset/System names known to the system (DVI-590). Gathers EAS values from every Preventative Maintenance work order sidecar (any status) plus the device-manual association index, so the WOR form can offer them as type-ahead suggestions. Deduplicated case-insensitively, preserving the first-seen display spelling, and sorted alphabetically. """ seen: dict[str, str] = {} def _add(raw: str) -> None: name = (raw or "").strip() if not name: return key = _normalize_equipment(name) if key and key not in seen: seen[key] = name for directory in _wor_type_dirs("preventive"): if not directory.is_dir(): continue for sidecar in directory.glob("*.json"): try: data = json.loads(sidecar.read_text()) except (json.JSONDecodeError, OSError): continue if isinstance(data, dict): _add(data.get("equipment", "")) for entry in _load_manuals_index().values(): if isinstance(entry, dict): _add(entry.get("equipment", "")) return sorted(seen.values(), key=lambda s: s.lower()) def _pm_process_manual(equipment: str, user: dict): """Resolve the manual for a PM submission from the request. Returns (manual_dict | None, error_message | None). A newly uploaded file (request.files['manual']) takes precedence and is saved + indexed; otherwise an existing manual referenced via the 'manual_existing' form field is reused (and re-associated with the current equipment). No manual -> (None, None). """ f = request.files.get("manual") if f and f.filename: ext = os.path.splitext(f.filename)[1].lower() if ext not in ALLOWED_MANUAL_EXTS: allowed = ", ".join(sorted(ALLOWED_MANUAL_EXTS)) return None, f"Unsupported manual file type. Allowed: {allowed}" WOR_MANUALS_DIR.mkdir(parents=True, exist_ok=True) base = secure_filename(f.filename) or ("manual" + ext) stored = f"{_datetime_type.now().strftime('%Y%m%d_%H%M%S')}_{base}" f.save(str(WOR_MANUALS_DIR / stored)) _manuals_index_set(equipment, stored, f.filename, user) return {"filename": stored, "original_name": f.filename, "url": _manual_viewer_url(stored)}, None existing = request.form.get("manual_existing", "").strip() if existing: if "/" in existing or "\\" in existing or ".." in existing: return None, "Invalid manual reference" if (WOR_MANUALS_DIR / existing).is_file(): original = request.form.get("manual_existing_name", "").strip() or existing _manuals_index_set(equipment, existing, original, user) return {"filename": existing, "original_name": original, "url": _manual_viewer_url(existing)}, None return None, None # --- Searchable manuals: OCR text index + background OCR (DVI-491) ------------ def _load_manuals_text_index() -> dict: """Return the {stored-filename: {text, pages, ...}} OCR text index.""" if WOR_MANUALS_TEXT_INDEX_FILE.is_file(): try: data = json.loads(WOR_MANUALS_TEXT_INDEX_FILE.read_text()) if isinstance(data, dict): return data except (json.JSONDecodeError, OSError): pass return {} def _save_manuals_text_index(index: dict) -> None: WOR_MANUALS_TEXT_INDEX_FILE.write_text(json.dumps(index, indent=2)) def _manuals_page_offsets(text: str) -> list: """Per-page [start, end) character offsets from the OCRmyPDF sidecar text. OCRmyPDF separates pages with a form feed (\\f); each page's slice lets the search route map a hit's character offset back to a 1-based page number. """ pages = [] pos = 0 for i, part in enumerate(text.split("\f")): start = pos end = pos + len(part) pages.append({"number": i + 1, "start": start, "end": end}) pos = end + 1 # account for the \f delimiter we split on return pages def _manuals_text_index_set(stored_filename: str, text: str, original_name: str) -> None: index = _load_manuals_text_index() index[stored_filename] = { "filename": stored_filename, "original_name": original_name, "text": text, "pages": _manuals_page_offsets(text), "indexed_at": _datetime_type.now().strftime("%Y-%m-%dT%H:%M:%S"), } _save_manuals_text_index(index) def _run_manual_ocr(stored_filename: str, original_name: str) -> None: """Background pass: replace a stored PDF manual with its searchable version and record the recognized text in the manuals text index. Only PDFs are processed (the only thing OCRmyPDF can turn into a searchable PDF in place without changing the stored filename the association index keys on). Non-PDF manuals are left untouched and unindexed. Engine-unavailable and OCR errors are swallowed so an upload never fails on a best-effort index pass. """ try: src = WOR_MANUALS_DIR / stored_filename if not src.is_file() or src.suffix.lower() != ".pdf": return try: from ocr_engine import ocr_pdf, OcrEngineUnavailable, SKIP_TEXT except ImportError: return tmp_out = src.with_suffix(".ocrtmp.pdf") sidecar = tmp_out.with_suffix(".txt") try: result = ocr_pdf(src, tmp_out, mode=SKIP_TEXT) except OcrEngineUnavailable: return except Exception: # noqa: BLE001 — OCR is best-effort; never break upload tmp_out.unlink(missing_ok=True) sidecar.unlink(missing_ok=True) return # Swap the searchable PDF in for the original so in-document Ctrl-F works. os.replace(str(tmp_out), str(src)) sidecar.unlink(missing_ok=True) _manuals_text_index_set(stored_filename, result.text, original_name) except Exception: # noqa: BLE001 — daemon thread must not crash the app return def _enqueue_manual_ocr(stored_filename: str, original_name: str) -> None: """Start a daemon thread to OCR + index a freshly uploaded manual (D3).""" threading.Thread( target=_run_manual_ocr, args=(stored_filename, original_name), daemon=True, ).start() def _backfill_manuals_ocr() -> None: """One-time idempotent pass: OCR + index any existing PDF manual not yet in the text index. Runs in a daemon thread at startup so it never blocks boot.""" try: indexed = set(_load_manuals_text_index().keys()) # Cover every PDF on disk, not just association-index entries, so manuals # attached to work orders are searchable too. if not WOR_MANUALS_DIR.is_dir(): return names = {} for entry in _load_manuals_index().values(): fn = entry.get("filename") if fn: names[fn] = entry.get("original_name", fn) for path in WOR_MANUALS_DIR.glob("*.pdf"): fn = path.name if fn in indexed: continue _run_manual_ocr(fn, names.get(fn, fn)) except Exception: # noqa: BLE001 — backfill is best-effort return # --------------------------------------------------------------------------- # Equipment Manuals: Grid-view thumbnail cache (DVI-585) # --------------------------------------------------------------------------- # The Grid view used to download each full manual and render page 1 client-side # with pdf.js, which is slow on the first load. We instead pre-render a small # JPEG per manual once, server-side, and cache it on disk. The grid then loads a # tiny ; the client-side path remains as a graceful fallback when the # renderers (PyMuPDF / Pillow) are unavailable. _MANUAL_THUMB_IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "webp", "bmp"} def _manual_thumb_path(stored_filename: str) -> Path: return WOR_MANUALS_THUMBS_DIR / (stored_filename + ".jpg") def _generate_manual_thumb(src: Path, ext: str, thumb: Path) -> Path | None: """Render a small JPEG thumbnail for one manual. Returns the path or None. Writes atomically (temp + replace) so a concurrent reader never sees a partial file. Best-effort: any renderer/IO error yields None so the grid falls back to client-side rendering. """ try: WOR_MANUALS_THUMBS_DIR.mkdir(parents=True, exist_ok=True) if ext == "pdf": import fitz # PyMuPDF — already a dependency of the Claude OCR backend import io doc = fitz.open(str(src)) try: if doc.page_count < 1: return None page = doc.load_page(0) zoom = MANUAL_THUMB_MAX_PX / max(page.rect.width, page.rect.height, 1) pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom)) img = _pil_image.open(io.BytesIO(pix.tobytes("png"))).convert("RGB") finally: doc.close() elif ext in _MANUAL_THUMB_IMAGE_EXTS: img = _pil_image.open(str(src)).convert("RGB") else: return None # doc/docx and anything else: no server thumbnail img.thumbnail((MANUAL_THUMB_MAX_PX, MANUAL_THUMB_MAX_PX)) tmp = thumb.with_suffix(".tmp.jpg") img.save(str(tmp), "JPEG", quality=82) os.replace(str(tmp), str(thumb)) return thumb except Exception: # noqa: BLE001 — thumbnailing is best-effort return None def _ensure_manual_thumb(stored_filename: str) -> Path | None: """Return a cached thumbnail path for a manual, regenerating on staleness. The cache is keyed by stored filename and invalidated by source mtime, so a manual replaced in place (e.g. by the OCR pass) re-renders on next request. """ src = WOR_MANUALS_DIR / stored_filename if not src.is_file(): return None thumb = _manual_thumb_path(stored_filename) try: if thumb.is_file() and thumb.stat().st_mtime >= src.stat().st_mtime: return thumb except OSError: pass ext = src.suffix.lower().lstrip(".") return _generate_manual_thumb(src, ext, thumb) def _backfill_manual_thumbs() -> None: """Pre-cache Grid-view thumbnails for every manual on disk (DVI-585). Runs in a daemon thread at startup so the first Grid-view load is served from cache instead of rendering each manual on demand. Best-effort.""" try: if not WOR_MANUALS_DIR.is_dir(): return for path in WOR_MANUALS_DIR.iterdir(): if not path.is_file(): continue ext = path.suffix.lower().lstrip(".") if ext == "pdf" or ext in _MANUAL_THUMB_IMAGE_EXTS: _ensure_manual_thumb(path.name) except Exception: # noqa: BLE001 — backfill is best-effort return def _load_branding_settings(): """Return branding settings, falling back to Togen defaults.""" settings = dict(DEFAULT_BRANDING_SETTINGS) if BRANDING_SETTINGS_FILE.is_file(): try: saved = json.loads(BRANDING_SETTINGS_FILE.read_text()) if isinstance(saved, dict): if saved.get("logo_mode") in BRANDING_LOGO_MODES: settings["logo_mode"] = saved["logo_mode"] if saved.get("bg_mode") in BRANDING_BG_MODES: settings["bg_mode"] = saved["bg_mode"] # Migration (DVI-366): seed each new theme-specific key from a # legacy single bg_opacity when the new key is absent. One-time # read only — we never write the old key back. legacy = _coerce_bg_opacity(saved.get("bg_opacity")) for key in ("bg_opacity_light", "bg_opacity_dark"): opacity = _coerce_bg_opacity(saved.get(key)) if opacity is None and key not in saved and legacy is not None: opacity = legacy if opacity is not None: settings[key] = opacity tile_columns = _coerce_tile_columns(saved.get("tile_columns")) if tile_columns is not None: settings["tile_columns"] = tile_columns tile_side_padding = _coerce_tile_side_padding(saved.get("tile_side_padding")) if tile_side_padding is not None: settings["tile_side_padding"] = tile_side_padding except (json.JSONDecodeError, OSError): pass return settings def _save_branding_settings(settings): BRANDING_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) def _get_qr_base_url(): """Return the base URL for QR-linked files: shared URL prefix + /shared path.""" prefix = _load_shared_url_prefix() return f"{prefix}/shared" _SAFE_JOB_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9 _\-\.]{0,79}$") # --------------------------------------------------------------------------- # Shared helpers # --------------------------------------------------------------------------- WOR_COUNTER_PATH = WOR_DIR / "wor_counter.json" def _detect_wor_type(file_path: Path) -> str: """Detect wor_type from the resolved file path.""" s = str(file_path.resolve()) if str(WOR_MOBILEMACHINE_DIR.resolve()) in s: return "mobilemachine" if str(WOR_SAFETY_DIR.resolve()) in s: return "safety" if str(WOR_PREVENTIVE_DIR.resolve()) in s: return "preventive" return "maintenance" def _wor_dest_dirs(wor_type: str) -> dict[str, Path]: """Return status->directory mapping for a WOR type.""" if wor_type == "mobilemachine": return { "new": WOR_MOBILEMACHINE_DIR, "in_progress": WOR_MM_INPROGRESS_DIR, "completed": WOR_MM_COMPLETED_DIR, "cancelled": WOR_MM_CANCELLED_DIR, } if wor_type == "safety": return { "new": WOR_SAFETY_DIR, "in_progress": WOR_SAFETY_INPROGRESS_DIR, "completed": WOR_SAFETY_COMPLETED_DIR, "cancelled": WOR_SAFETY_CANCELLED_DIR, } if wor_type == "preventive": return { "new": WOR_PREVENTIVE_DIR, "in_progress": WOR_PREVENTIVE_INPROGRESS_DIR, "completed": WOR_PREVENTIVE_COMPLETED_DIR, "cancelled": WOR_PREVENTIVE_CANCELLED_DIR, } return { "new": WOR_MAINTENANCE_DIR, "in_progress": WOR_MAINT_INPROGRESS_DIR, "completed": WOR_MAINT_COMPLETED_DIR, "cancelled": WOR_MAINT_CANCELLED_DIR, } # Lifecycle status -> notification event. "new" is not a notifiable transition. _WOR_STATUS_EVENTS = { "in_progress": notifications.EVENT_WO_STATUS_IN_PROGRESS, "completed": notifications.EVENT_WO_STATUS_COMPLETED, "cancelled": notifications.EVENT_WO_STATUS_CANCELLED, } def _wor_event_context(filename: str, wor_type: str, sidecar_data: dict) -> dict: """Build the notify() context for a work-order event from its sidecar. Carries recipient-resolution inputs (per-WO opt-in ``_notify``, the submitter email, and ``wor_type``) plus the template fields the branded email renders. ``wor_id`` is the filename — stable across status moves so the idempotency log keys correctly. """ sd = sidecar_data or {} return { "wor_id": filename, "wor_number": sd.get("_wor_number"), "wor_type": wor_type, "notify": sd.get("_notify"), "submitter_email": sd.get("email"), "email": sd.get("email"), "requestor_name": sd.get("requestor_name"), "location": sd.get("location"), "building": sd.get("building"), "urgency": sd.get("urgency"), "desired_completion_date": sd.get("desired_completion_date"), "description": sd.get("description"), } def _next_work_order_number() -> int: """Atomically increment and return the next work order number.""" WOR_DIR.mkdir(parents=True, exist_ok=True) counter = 0 if WOR_COUNTER_PATH.exists(): try: counter = json.loads(WOR_COUNTER_PATH.read_text()).get("next_number", 0) except (json.JSONDecodeError, OSError): counter = 0 counter = max(counter, 0) + 1 WOR_COUNTER_PATH.write_text(json.dumps({"next_number": counter})) return counter def _load_recurring_registry() -> dict: """Load the recurring work order registry.""" if WOR_RECURRING_PATH.exists(): try: return json.loads(WOR_RECURRING_PATH.read_text()) except (json.JSONDecodeError, OSError): pass return {} def _save_recurring_registry(registry: dict): """Save the recurring work order registry.""" WOR_DIR.mkdir(parents=True, exist_ok=True) WOR_RECURRING_PATH.write_text(json.dumps(registry, indent=2)) def _calc_next_run(start_date: str, frequency_days: int, after_date: str | None = None) -> str | None: """Calculate the next run date from start_date + frequency, on or after after_date.""" from datetime import timedelta try: start = _datetime_type.strptime(start_date, "%Y-%m-%d").date() except (ValueError, TypeError): return None ref = _datetime_type.now().date() if not after_date else _datetime_type.strptime(after_date, "%Y-%m-%d").date() delta = timedelta(days=frequency_days) nxt = start while nxt <= ref: nxt += delta return nxt.strftime("%Y-%m-%d") def _recurring_runs_in_window(start_date: str, frequency_days: int, end_date: str, win_start, win_end) -> list: """Enumerate recurring run dates (date objects) falling within [win_start, win_end]. ``start_date``/``end_date`` are ``YYYY-MM-DD`` strings (end optional); ``win_start``/``win_end`` are ``date`` objects. Used by the Calendar tool to plot a marker on every due date of a recurring schedule (DVI-630). """ from datetime import timedelta runs: list = [] try: cur = _datetime_type.strptime(start_date, "%Y-%m-%d").date() except (ValueError, TypeError): return runs if not frequency_days or frequency_days <= 0: return runs end = None if end_date: try: end = _datetime_type.strptime(end_date, "%Y-%m-%d").date() except (ValueError, TypeError): end = None delta = timedelta(days=frequency_days) # Fast-forward to the first run on or after the window start. if cur < win_start: skip = (win_start - cur).days // frequency_days cur = cur + delta * skip while cur < win_start: cur += delta while cur <= win_end: if end and cur > end: break runs.append(cur) cur += delta return runs WOR_ADMIN_EMAIL = "workorderadmins@icastinc.com" def _send_wor_admin_email(subject: str, body_text: str): """Send a notification email to WOR admins via Microsoft Graph API. Fire-and-forget.""" if not AZURE_MAIL_FROM: app.logger.warning("Cannot send WOR admin email: AZURE_MAIL_FROM not set") return try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: app.logger.error("WOR admin email: failed to acquire token") return graph_body = { "message": { "subject": subject, "body": {"contentType": "Text", "content": body_text}, "toRecipients": [ {"emailAddress": {"address": WOR_ADMIN_EMAIL}}, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code not in (200, 202): app.logger.error("WOR admin email failed: %s", resp.text[:300]) except Exception: app.logger.exception("WOR admin email error") def _send_wor_admin_email_bg(subject: str, body_text: str): """Send admin email in a background thread to avoid blocking requests.""" t = threading.Thread(target=_send_wor_admin_email, args=(subject, body_text), daemon=True) t.start() def _resolve_wor_sidecar(filename: str, wor_type: str | None = None) -> Path | None: """Find the JSON sidecar for a WOR PDF across all directories.""" json_name = Path(filename).with_suffix(".json").name if not json_name or ".." in json_name or "/" in json_name: return None for d in _wor_type_dirs(wor_type): candidate = (d / json_name).resolve() if not str(candidate).startswith(str(d.resolve())): return None if candidate.is_file(): return candidate return None def _wor_number_label(filename: str, data: dict | None) -> str: """Return a "WO #00042" label from sidecar data, falling back to the filename.""" num = (data or {}).get("_wor_number") if isinstance(num, int): return f"WO #{num:05d}" m = re.search(r"_(\d{4,6})_", filename or "") if m: return f"WO #{int(m.group(1)):05d}" return "WO" def _wor_abbrev(text: str, limit: int = 40) -> str: """Collapse whitespace and truncate request text, appending an ellipsis.""" text = " ".join((text or "").split()) if len(text) <= limit: return text return text[:limit].rstrip() + "..." def _wor_display_title(filename: str, wor_type: str, data: dict | None) -> str: """Build a human-friendly WOR list title from sidecar data (DVI-393, DVI-394). Format: "WO #NNNNN - - " where description is the machine + asset number for mobile-machine WORs, or the abbreviated request text otherwise. The WOR type is rendered separately as a list tag, so it is intentionally not repeated in the title. Falls back to the filename when no sidecar data is available. """ if not data: return filename wo = _wor_number_label(filename, data) if wor_type == "mobilemachine": requestor = (data.get("_notify") or {}).get("requestor_name") or "" desc = " ".join(s for s in (data.get("machine"), data.get("asset_number")) if s) else: requestor = (data.get("requestor_name") or (data.get("_notify") or {}).get("requestor_name") or "") desc = _wor_abbrev(data.get("description") or "") segments = [wo] + ([desc] if desc else []) + ([requestor] if requestor else []) return " - ".join(segments) def _extract_wor_data_from_pdf(pdf_path: Path) -> dict: """Best-effort extraction of form data from a WOR PDF (fallback when no sidecar).""" from pypdf import PdfReader reader = PdfReader(str(pdf_path)) text = reader.pages[0].extract_text() if reader.pages else "" lines = text.split("\n") data: dict = {} for i, line in enumerate(lines): s = line.strip() if s.startswith("Requestor name"): if i + 1 < len(lines): val = lines[i + 1].strip() email_match = re.search(r'[\w.\-+]+@[\w.\-]+\.\w+', val) if email_match: data["email"] = email_match.group(0) val = val[:email_match.start()].strip() phone_match = re.search(r'(\d[\d\-\(\)\s\.]{6,}\d)', val) if phone_match: data["phone"] = phone_match.group(1).strip() data["requestor_name"] = val[:phone_match.start()].strip() else: data["requestor_name"] = val elif s.startswith("Date of request"): if i + 1 < len(lines): val = lines[i + 1].strip() date_match = re.match(r'(\d{4}-\d{2}-\d{2})', val) if date_match: data["date_of_request"] = date_match.group(1) data["location"] = val[date_match.end():].strip() else: data["date_of_request"] = val elif s.startswith("Building"): if i + 1 < len(lines): data["building"] = lines[i + 1].strip() elif s == "Request description": desc = [] for j in range(i + 1, len(lines)): if lines[j].strip().startswith("Type of issue"): break desc.append(lines[j]) data["description"] = "\n".join(desc).strip() elif s.startswith("Type of issue"): types = [] all_types = [ "Automotive", "Grounds", "Plumbing", "Custodial", "Cranes", "Concrete Forms", "Telehandler/Forklift", "Machine", "Electrical", "Tool Repair", "Safety", "Other", ] for j in range(i + 1, min(i + 8, len(lines))): cl = lines[j] if cl.strip().startswith("Best times") or cl.strip().startswith("Urgency"): break for t in all_types: if "✔" in cl and t in cl: idx_check = cl.find("✔") idx_type = cl.find(t) if idx_check < idx_type and (idx_type - idx_check) < 15: types.append(t) # Simpler fallback: just look for ✔ directly before each type name if not types: block = "\n".join(lines[i + 1: i + 8]) for t in all_types: if re.search(r"[✔4]\s*" + re.escape(t), block): types.append(t) data["issue_type"] = types elif s == "Best times for service": if i + 1 < len(lines): data["best_times"] = lines[i + 1].strip() elif s.startswith("Desired completion date"): if i + 1 < len(lines): data["desired_completion_date"] = lines[i + 1].strip() elif s.startswith("Urgency"): for j in range(i + 1, min(i + 5, len(lines))): if "✔" in lines[j] or "4" == lines[j].strip()[:1]: urg = lines[j].replace("✔", "").strip() if urg in ("Low", "Medium", "High"): data["urgency"] = urg break return data def _safe_job_folder(job_id: str) -> Path: """Return resolved Path for job_id; abort(400) if unsafe.""" if not _SAFE_JOB_RE.match(job_id) or ".." in job_id: abort(400) folder = (WATCH_DIR / job_id).resolve() if not str(folder).startswith(str(WATCH_DIR.resolve())): abort(400) return folder def _find_generated(job_folder: Path) -> Path | None: if not job_folder.is_dir(): return None for f in job_folder.iterdir(): if f.is_file() and "generated" in f.name.lower() and f.suffix.lower() == ".pdf": return f return None def _state_path(folder: Path) -> Path: return folder / "state.json" def _load_state(folder: Path) -> dict: p = _state_path(folder) return json.loads(p.read_text()) if p.exists() else {} def _save_state(folder: Path, state: dict) -> None: _state_path(folder).write_text(json.dumps(state, indent=2)) def _match_files_to_items( folder: Path, items: list[str], item_refs: list[str], item_cols_a: list[str], ) -> list[dict]: """Scan *folder* recursively for PDFs and match them to items by identifier presence + suffix. Identifiers: col A (item_cols_a[i]), col H (item_refs[i]), col C (items[i]). All non-empty identifiers must appear in the filename stem (case-insensitive). Suffix determines file type: stem ends with 'shop' → shop drawing stem ends with 'submittal' → submittal document stem ends with 'carrierqr' → QR carrier sheet Submittal fallback: if no submittal matches all identifiers (common when the submittal is a shared document not named per item), retry with only col A as the required identifier. Paths stored are relative to *folder* so they work regardless of subdirectory depth. Returns a list[dict] parallel to *items*; each dict maps 'shop'/'submittal'/'carrier' to the relative path string from *folder* (key absent if no match found). """ # rglob so files inside subdirectories (e.g. zip-extracted sub-folder) are found pdf_files = [f for f in folder.rglob("*.pdf")] auto_files: list[dict] = [] for i, item_name in enumerate(items): col_a = item_cols_a[i] if i < len(item_cols_a) else "" col_h = item_refs[i] if i < len(item_refs) else "" col_c = item_name # All non-empty identifiers must appear in the stem (case-insensitive) identifiers = [s.strip().lower() for s in [col_a, col_h, col_c] if s.strip()] col_a_only = [col_a.strip().lower()] if col_a.strip() else [] matched: dict = {} for pdf in pdf_files: stem_lower = pdf.stem.lower() rel = str(pdf.relative_to(folder)) if not all(ident in stem_lower for ident in identifiers): continue if stem_lower.endswith("shop"): matched.setdefault("shop", rel) elif stem_lower.endswith("submittal"): matched.setdefault("submittal", rel) elif stem_lower.endswith("carrierqr"): matched.setdefault("carrier", rel) # Submittal fallback: shared submittal files often omit item-specific identifiers. # Only apply when a shop drawing was also matched — no shop means no submittal. if "submittal" not in matched and "shop" in matched and col_a_only: for pdf in pdf_files: stem_lower = pdf.stem.lower() if stem_lower.endswith("submittal") and all(ident in stem_lower for ident in col_a_only): matched.setdefault("submittal", str(pdf.relative_to(folder))) break auto_files.append(matched) return auto_files # --------------------------------------------------------------------------- # Legacy single-item background merge (unchanged from original) # --------------------------------------------------------------------------- def _run_merge_single(job_folder: Path) -> None: """Classify uploaded PDFs by filename and run the QR merge (legacy route).""" if build_carrier_pdf is None: return carrier = submittal = shop = None for f in sorted(job_folder.iterdir()): if not f.is_file() or f.suffix.lower() != ".pdf": continue name = f.name.lower() if "generated" in name: continue if "carrier" in name and carrier is None: carrier = f elif "submittal" in name and submittal is None: submittal = f elif "shop" in name and shop is None: shop = f if not (carrier and submittal and shop): return folder_name = job_folder.name _qr_base = _get_qr_base_url() folder_url = f"{_qr_base.rstrip('/')}/{quote(folder_name)}" submittal_url = f"{folder_url}/{quote(submittal.name)}" shop_url = f"{folder_url}/{quote(shop.name)}" blank_bytes = carrier.read_bytes() result_bytes = build_carrier_pdf(blank_bytes, submittal_url, shop_url) output_path = job_folder / f"{folder_name} CarrierQR generated.pdf" output_path.write_bytes(result_bytes) # --------------------------------------------------------------------------- # HTML (multi-item CSV workflow at top; legacy single-item below) # --------------------------------------------------------------------------- _HTML = r""" Togen — Pull it Forward

{{ site_title|e }}

{{ site_tagline|e }}
""" # --------------------------------------------------------------------------- # Authentication helpers — Azure AD SSO (DVI-194) # --------------------------------------------------------------------------- def _is_json_request(): return request.is_json or "application/json" in (request.headers.get("Accept") or "") def login_required(f): """Redirect unauthenticated requests to /login (return JSON 401 for API calls).""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) return f(*args, **kwargs) return _wrap def wor_admin_required(f): """Require login + WOR Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) if not session["user"].get("wor_admin"): return jsonify(error="Access denied: Work Order Admin group membership required."), 403 return f(*args, **kwargs) return _wrap def togen_admin_required(f): """Require login + Togen Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) if not session["user"].get("togen_admin"): return jsonify(error="Access denied: Togen Admin group membership required."), 403 return f(*args, **kwargs) return _wrap def takeoff_workflow_required(f): """Require login + Togen Take-off Workflow or Togen Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if not (u.get("takeoff_workflow") or u.get("togen_admin")): return jsonify(error="Access denied: Togen Take-off Workflow group membership required."), 403 return f(*args, **kwargs) return _wrap def prod_run_required(f): """Require login + Togen Production Run or Togen Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if not (u.get("prod_run") or u.get("togen_admin")): return jsonify(error="Access denied: Togen Production Run group membership required."), 403 return f(*args, **kwargs) return _wrap def mts_label_required(f): """Require login + Togen MTS or Togen Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if not (u.get("togen_mts") or u.get("togen_admin")): return jsonify(error="Access denied: Togen MTS group membership required."), 403 return f(*args, **kwargs) return _wrap def scada_required(f): """Require login + Togen SCADA or Togen Admin group membership.""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if not (u.get("togen_scada") or u.get("togen_admin")): return jsonify(error="Access denied: Togen SCADA group membership required."), 403 return f(*args, **kwargs) return _wrap def vision_access_required(f): """Require login + Vision access: togen_admin always, OR togen_vision only while the Vision tool is enabled in Tools Enabled (admin-disabled => 403).""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if u.get("togen_admin"): return f(*args, **kwargs) if u.get("togen_vision") and _load_tool_visibility().get("vision", False): return f(*args, **kwargs) return jsonify(error="Access denied: Vision access required."), 403 return _wrap def rpm_required(f): """Require Repair Parts Manager access (DVI-596): togen_admin always, OR wor_admin only while the tool is enabled in Tools Enabled (admin-disabled => 403).""" @wraps(f) def _wrap(*args, **kwargs): if not session.get("user"): if _is_json_request(): return jsonify(error="Session expired. Please reload the page and log in again."), 401 session["next"] = request.url return redirect(url_for("login")) u = session["user"] if u.get("togen_admin"): return f(*args, **kwargs) if u.get("wor_admin") and _load_tool_visibility().get("repair-parts-manager", False): return f(*args, **kwargs) return jsonify(error="Access denied: Repair Parts Manager access required."), 403 return _wrap def _wor_type_dirs(wor_type: str | None = None) -> list[Path]: """Return the ordered list of directories to search for a given WOR type.""" if wor_type == "mobilemachine": return [WOR_MOBILEMACHINE_DIR, WOR_MM_INPROGRESS_DIR, WOR_MM_COMPLETED_DIR, WOR_MM_CANCELLED_DIR] if wor_type == "safety": return [WOR_SAFETY_DIR, WOR_SAFETY_INPROGRESS_DIR, WOR_SAFETY_COMPLETED_DIR, WOR_SAFETY_CANCELLED_DIR] if wor_type == "preventive": return [WOR_PREVENTIVE_DIR, WOR_PREVENTIVE_INPROGRESS_DIR, WOR_PREVENTIVE_COMPLETED_DIR, WOR_PREVENTIVE_CANCELLED_DIR] if wor_type == "maintenance": return [WOR_MAINTENANCE_DIR, WOR_MAINT_INPROGRESS_DIR, WOR_MAINT_COMPLETED_DIR, WOR_MAINT_CANCELLED_DIR] return [ WOR_MAINTENANCE_DIR, WOR_MAINT_INPROGRESS_DIR, WOR_MAINT_COMPLETED_DIR, WOR_MAINT_CANCELLED_DIR, WOR_MOBILEMACHINE_DIR, WOR_MM_INPROGRESS_DIR, WOR_MM_COMPLETED_DIR, WOR_MM_CANCELLED_DIR, WOR_SAFETY_DIR, WOR_SAFETY_INPROGRESS_DIR, WOR_SAFETY_COMPLETED_DIR, WOR_SAFETY_CANCELLED_DIR, WOR_PREVENTIVE_DIR, WOR_PREVENTIVE_INPROGRESS_DIR, WOR_PREVENTIVE_COMPLETED_DIR, WOR_PREVENTIVE_CANCELLED_DIR, ] def _resolve_wor_file(filename: str, wor_type: str | None = None) -> Path | None: """Find a WOR file across new / in_progress / completed directories.""" if not filename or ".." in filename or "/" in filename: return None for d in _wor_type_dirs(wor_type): candidate = (d / filename).resolve() if not str(candidate).startswith(str(d.resolve())): return None if candidate.is_file(): return candidate return None @app.route("/login") def login(): """Initiate Microsoft OAuth 2.0 Authorization Code flow.""" cl = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY, client_credential=_AZ_SECRET) auth_url = cl.get_authorization_request_url( _AZ_SCOPES, redirect_uri=_AZ_REDIRECT, state=session.get("next", "/"), ) return redirect(auth_url) @app.route("/auth/callback") def auth_callback(): """Handle the redirect from Microsoft after login or admin consent.""" # Admin-consent redirects don't carry an authorization code. if request.args.get("admin_consent", "").lower() == "true": tenant = request.args.get("tenant", "unknown") return ( f"Admin consent granted successfully for tenant {tenant}. " "You may close this window." ), 200 code = request.args.get("code") if not code: return "Authentication failed: no code returned.", 401 cl = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY, client_credential=_AZ_SECRET) result = cl.acquire_token_by_authorization_code( code, scopes=_AZ_SCOPES, redirect_uri=_AZ_REDIRECT) if "error" in result: return f"Login error: {result.get('error_description', result['error'])}", 401 claims = result["id_token_claims"] token = result["access_token"] user_tenant = claims.get("tid", "") username = claims.get("preferred_username", "unknown") # Validate the user's tenant against the allowlist. tenant_cfg = _AZ_TENANT_CONFIG.get(user_tenant) if not tenant_cfg: print( f"[AUTH] Rejected login from unknown tenant {user_tenant} " f"(user={username})", flush=True, ) return ( "Access denied: your organization is not authorized to use Togen. " "Contact your IT administrator to request access.", 403, ) tenant_name = tenant_cfg["name"] base_group = tenant_cfg["base_group"] tenant_groups = tenant_cfg.get("groups", {}) print( f"[AUTH] Login from tenant {tenant_name} ({user_tenant}), " f"user={username}", flush=True, ) # Verify the user is a member of the tenant's base access group. # base_group=None means any user from the tenant is allowed (DVI-280). if base_group is not None: graph_resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/me/memberOf/{base_group}", headers={"Authorization": f"Bearer {token}"}, timeout=10, ) if graph_resp.status_code != 200: print( f"[AUTH] Base group check failed for {username}: " f"group={base_group}, status={graph_resp.status_code}", flush=True, ) return ( "Access denied: your account is not a member of the required " f"Togen group in {tenant_name}. " "Contact your IT administrator to request access.", 403, ) # Check role-specific group memberships based on tenant config. role_flags = {} for role_key in ("wor_admin", "togen_admin", "takeoff_workflow", "prod_run", "togen_mts", "togen_scada", "togen_vision"): group_id = tenant_groups.get(role_key) if not group_id: role_flags[role_key] = False continue try: resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/me/memberOf/{group_id}", headers={"Authorization": f"Bearer {token}"}, timeout=10, ) role_flags[role_key] = resp.status_code == 200 print( f"[{role_key.upper()}] Group check for {username}: " f"status={resp.status_code}, granted={role_flags[role_key]}", flush=True, ) except Exception as exc: role_flags[role_key] = False print(f"[{role_key.upper()}] Group check failed: {exc}", flush=True) session["user"] = { "name": claims.get("name"), "email": username, "tenant": tenant_name, "wor_admin": role_flags["wor_admin"], "togen_admin": role_flags["togen_admin"], "takeoff_workflow": role_flags["takeoff_workflow"], "prod_run": role_flags["prod_run"], "togen_mts": role_flags["togen_mts"], "togen_scada": role_flags["togen_scada"], "togen_vision": role_flags["togen_vision"], } next_url = request.args.get("state", "/") return redirect(next_url) @app.route("/logout") def logout(): """Clear local session and sign out from Microsoft.""" session.clear() post_logout = "https://togen.icastinc.com" return redirect( f"{_AZ_AUTHORITY}/oauth2/v2.0/logout" f"?post_logout_redirect_uri={post_logout}" ) # --------------------------------------------------------------------------- # Routes — multi-item CSV workflow # --------------------------------------------------------------------------- @app.route("/") @login_required def index(): from assistant import is_assistant_connected user = session.get("user", {}) user_email = user.get("email", "") _assistant_available = is_assistant_connected( user_email, user.get("togen_admin", False), _load_user_settings(user_email) ) _is_admin = user.get("togen_admin", False) _toolbar_vis = _load_toolbar_visibility() return render_template_string( _HTML, assistant_available=_assistant_available, toolbar_visibility=_toolbar_vis, show_toolbar_search=_toolbar_enabled_for("search", _is_admin, _toolbar_vis), show_toolbar_assistant=_toolbar_enabled_for("assistant", _is_admin, _toolbar_vis), show_toolbar_calendar=_toolbar_enabled_for("calendar", _is_admin, _toolbar_vis), user_name=user.get("name", "User"), user_email=user_email, user_tenant=user.get("tenant", ""), is_wor_admin=user.get("wor_admin", False), is_togen_admin=user.get("togen_admin", False), is_takeoff_workflow=user.get("takeoff_workflow", False), is_prod_run=user.get("prod_run", False), is_togen_mts=user.get("togen_mts", False), is_togen_scada=user.get("togen_scada", False), is_togen_vision=user.get("togen_vision", False), tool_visibility=_load_tool_visibility(), tool_order=_load_tool_order(), tool_overrides=_load_tool_overrides(), wor_type_visibility=_load_wor_type_visibility(), wor_type_order=_load_wor_type_order(), public_version=_load_public_version(), about_html=_render_news_markdown(_load_about_content()), site_title=_load_site_text()["title"], site_tagline=_load_site_text()["tagline"], shared_url_prefix=_load_shared_url_prefix(), user_settings=_user_settings_with_grid(user_email), wor_primary_enforced=_wor_primary_enforced_for(user_email), branding_settings=_load_branding_settings(), branding_has_logo=CORPORATE_LOGO_FILE.is_file(), print_can_use=_user_can_print(user), ) @app.route("/upload-csv", methods=["POST"]) @takeoff_workflow_required def upload_csv(): if "csv" not in request.files: return jsonify({"error": "No CSV file provided"}), 400 f = request.files["csv"] if not f.filename or not f.filename.lower().endswith(".csv"): return jsonify({"error": "File must be a .csv"}), 400 safe_name = secure_filename(f.filename) stem = Path(safe_name).stem folder = WATCH_DIR / stem folder.mkdir(parents=True, exist_ok=True) csv_path = folder / safe_name f.save(str(csv_path)) # Skip header row; use column C (index 2) as item name, column H (index 7) as part reference items: list[str] = [] item_refs: list[str] = [] with open(csv_path, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) next(reader, None) # header for row in reader: if len(row) >= 3 and row[2].strip(): items.append(row[2].strip()) item_refs.append(row[7].strip() if len(row) >= 8 else "") if not items: return jsonify({"error": "No items found — expected data in column C from row 2 onward"}), 400 state: dict = { "csv_filename": safe_name, "stem": stem, "items": items, "item_refs": item_refs, "files": {str(i): {} for i in range(len(items))}, "results": {}, } _save_state(folder, state) return jsonify({"folder": stem, "items": items, "item_refs": item_refs}) @app.route("/upload-zip", methods=["POST"]) @takeoff_workflow_required def upload_zip(): if "zip" not in request.files: return jsonify({"error": "No zip file provided"}), 400 f = request.files["zip"] if not f.filename or not f.filename.lower().endswith(".zip"): return jsonify({"error": "File must be a .zip"}), 400 safe_name = secure_filename(f.filename) stem = Path(safe_name).stem folder = WATCH_DIR / stem folder.mkdir(parents=True, exist_ok=True) # Extract zip, rejecting any entries that would escape the target folder with zipfile.ZipFile(f.stream) as zf: for member in zf.namelist(): dest = (folder / member).resolve() if not str(dest).startswith(str(folder.resolve())): return jsonify({"error": f"Unsafe path in zip entry: {member}"}), 400 zf.extractall(str(folder)) # Find the CSV file (search root first, then subdirectories) csv_files = sorted(folder.glob("*.csv")) or sorted(folder.glob("**/*.csv")) if not csv_files: return jsonify({"error": "No CSV file found in zip"}), 400 csv_path = csv_files[0] # Parse same as upload_csv: skip header, col A = identifier, col C = item name, col H = ref items: list[str] = [] item_refs: list[str] = [] item_cols_a: list[str] = [] with open(csv_path, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) next(reader, None) for row in reader: if len(row) >= 3 and row[2].strip(): items.append(row[2].strip()) item_refs.append(row[7].strip() if len(row) >= 8 else "") item_cols_a.append(row[0].strip() if row else "") if not items: return jsonify({"error": "CSV in zip has no items (expected data in column C from row 2 onward)"}), 400 # Auto-match extracted PDF files to CSV rows auto_files = _match_files_to_items(folder, items, item_refs, item_cols_a) # Pre-populate state["files"] with matched filenames files_state: dict = {} for i, matched in enumerate(auto_files): files_state[str(i)] = dict(matched) # copy; user uploads will override state: dict = { "csv_filename": str(csv_path.relative_to(folder)), "stem": stem, "items": items, "item_refs": item_refs, "item_cols_a": item_cols_a, "files": files_state, "results": {}, } _save_state(folder, state) return jsonify({"folder": stem, "items": items, "item_refs": item_refs, "auto_files": auto_files}) @app.route("/upload-file", methods=["POST"]) @takeoff_workflow_required def upload_file(): if "file" not in request.files: return jsonify({"error": "No file provided"}), 400 f = request.files["file"] if not f.filename or not f.filename.lower().endswith(".pdf"): return jsonify({"error": "Only PDF files are accepted"}), 400 folder_name = request.form.get("folder", "") item_index = request.form.get("item_index", "") file_type = request.form.get("file_type", "") if file_type not in ("shop", "submittal", "carrier"): return jsonify({"error": "Invalid file_type"}), 400 if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 folder = WATCH_DIR / folder_name if not folder.exists(): return jsonify({"error": "Session not found — please re-upload CSV"}), 400 save_name = secure_filename(f.filename) f.save(str(folder / save_name)) state = _load_state(folder) state.setdefault("files", {}).setdefault(item_index, {})[file_type] = save_name _save_state(folder, state) return jsonify({"ok": True}) @app.route("/detach-file", methods=["POST"]) @takeoff_workflow_required def detach_file(): """Disconnect a detected/uploaded file from an item without deleting it. Removes only the state.json association so /submit treats the slot as empty (e.g. detaching a carrier re-enables the Create path). Reopening the folder re-runs auto-matching, so a detach lasts for the session. """ data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") item_index = str(data.get("item_index", "")) file_type = data.get("file_type", "") if file_type not in ("shop", "submittal", "carrier"): return jsonify({"error": "Invalid file_type"}), 400 if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 folder = WATCH_DIR / folder_name if not folder.exists(): return jsonify({"error": "Session not found — please re-upload CSV"}), 400 state = _load_state(folder) item_files = state.get("files", {}).get(item_index) if isinstance(item_files, dict): item_files.pop(file_type, None) if file_type == "carrier": item_files.pop("created", None) if not item_files: state["files"].pop(item_index, None) _save_state(folder, state) return jsonify({"ok": True}) # Project-metadata keys rendered onto a generated carrier sheet (vendor_project_code # is saved in project_metadata.json but has no field on the carrier template). _CARRIER_META_KEYS = ( "job_number", "production_date", "spec_type", "station", "customer_name", "project_name", ) def _build_carrier_fields(row: list, metadata: dict, override: dict) -> dict: """Fields dict for qr_merge.create_carrier_sheet from a CSV row + project metadata. CSV columns: B=structure_id, C=product_code, F=weight, G=structure_size, H=part_ref. Per-item override values win over project-level metadata. """ def col(idx: int) -> str: return row[idx].strip() if len(row) > idx else "" fields = { "structure_id": col(1), "product_code": col(2), "weight": col(5), "structure_size": col(6), "part_ref": col(7), } for key in _CARRIER_META_KEYS: fields[key] = str(override.get(key, metadata.get(key, "")) or "").strip() return fields @app.route("/submit", methods=["POST"]) @takeoff_workflow_required def submit(): data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 folder = WATCH_DIR / folder_name state = _load_state(folder) items = state.get("items", []) if not items: return jsonify({"error": "No items in session — please re-upload CSV"}), 400 raw_flags = data.get("create_flags") create_flags = ( {str(k): bool(v) for k, v in raw_flags.items()} if isinstance(raw_flags, dict) else {} ) # Project metadata for generated sheets: payload first, project_metadata.json fallback metadata = data.get("metadata") if isinstance(data.get("metadata"), dict) else None if metadata is None and any(create_flags.values()): meta_path = folder / "project_metadata.json" if meta_path.is_file(): try: loaded = json.loads(meta_path.read_text(encoding="utf-8")) if isinstance(loaded, dict): metadata = loaded except (json.JSONDecodeError, OSError): pass metadata = metadata or {} item_overrides = metadata.get("item_overrides") if not isinstance(item_overrides, dict): item_overrides = {} _create_carrier_sheet = None _create_import_error = "" if any(create_flags.values()): try: from qr_merge import create_carrier_sheet as _create_carrier_sheet # noqa: E402 except ImportError: _create_import_error = ( "Carrier sheet generation is unavailable on this host " "(qr_merge.create_carrier_sheet not found)" ) # Load CSV rows to append URL columns csv_path = folder / state["csv_filename"] with open(csv_path, newline="", encoding="utf-8-sig") as fh: rows = list(csv.reader(fh)) # Map item index → CSV row index (skip header; only count populated col-C rows) item_row_map: dict[int, int] = {} item_count = 0 for row_idx, row in enumerate(rows): if row_idx == 0: continue if len(row) >= 3 and row[2].strip(): item_row_map[item_count] = row_idx item_count += 1 results: list[dict] = [] for i, item_name in enumerate(items): files = state.get("files", {}).get(str(i), {}) shop_name = files.get("shop") carrier_name = files.get("carrier") submittal_name = files.get("submittal") # No shop drawing → skip this line (no merge, no CSV URLs written) if not shop_name: results.append({"skipped": True}) continue # No carrier sheet: generate one when the item is create-flagged, # otherwise keep the existing validation error. created_now = False if not carrier_name: if not create_flags.get(str(i)): results.append({"error": f'Shop Drawing uploaded for "{item_name}" but QR Carrier Sheet is required'}) continue if _create_import_error: results.append({"error": _create_import_error}) continue row = rows[item_row_map[i]] if i in item_row_map else [] fields = _build_carrier_fields(row, metadata, item_overrides.get(str(i), {})) if not fields["customer_name"] or not fields["project_name"]: results.append({"error": f'Project metadata (customer and project name) is required to create a carrier sheet for "{item_name}"'}) continue try: gen_path = _create_carrier_sheet(fields=fields, output_folder=folder) # The intermediate is named "... CarrierQR blank.pdf"; the merged # result is final, so name it like SCS output ("CarrierQR generated"). final_path = folder / gen_path.name.replace("CarrierQR blank", "CarrierQR generated") if final_path != gen_path: gen_path.replace(final_path) carrier_name = final_path.name except Exception as exc: results.append({"error": f'Carrier sheet generation failed for "{item_name}": {exc}'}) continue created_now = True item_files = state.setdefault("files", {}).setdefault(str(i), {}) item_files["carrier"] = carrier_name item_files["created"] = True shop_path = folder / shop_name carrier_path = folder / carrier_name submittal_path = (folder / submittal_name) if submittal_name else None try: output_path = _merge_item(carrier_path, shop_path, folder, submittal_path) # shop_name / carrier_name / submittal_name are relative paths from folder # (either just "file.pdf" for manual uploads or "subdir/file.pdf" for zip auto-matches). # Use safe='/' so subdirectory separators survive URL encoding. _qr_base = _get_qr_base_url() folder_url = f"{_qr_base.rstrip('/')}/{quote(folder_name, safe='')}" shop_url = f"{folder_url}/{quote(shop_name, safe='/')}" carrier_url = f"{folder_url}/{quote(carrier_name, safe='/')}" submittal_url = f"{folder_url}/{quote(submittal_name, safe='/')}" if submittal_name else "" # Write URLs into columns J (9), K (10), L (11) if i in item_row_map: row = rows[item_row_map[i]] while len(row) < 12: row.append("") row[9] = shop_url row[10] = submittal_url row[11] = carrier_url state.setdefault("results", {})[str(i)] = {"output": carrier_name} result = { "ok": True, "download_url": f"download-file?folder={quote(folder_name, safe='')}&file={quote(carrier_name, safe='/')}", "preview_url": f"preview-file?folder={quote(folder_name, safe='')}&file={quote(carrier_name, safe='/')}", } if created_now: result["created"] = True result["carrier"] = carrier_name results.append(result) except Exception as exc: results.append({"error": str(exc)}) # Overwrite the original CSV with the updated rows (URLs now in columns J/K/L) with open(csv_path, "w", newline="", encoding="utf-8") as fh: csv.writer(fh).writerows(rows) if create_flags: state["create_flags"] = create_flags _save_state(folder, state) return jsonify({"results": results}) @app.route("/download-file") @takeoff_workflow_required def download_file(): folder_name = request.args.get("folder", "") filename = request.args.get("file", "") if not folder_name or not filename: return "Missing parameters", 400 if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return "Invalid folder", 400 # Use path resolution for safety instead of secure_filename, which mangles spaces. folder = (WATCH_DIR / folder_name).resolve() file_path = (folder / filename).resolve() if not str(file_path).startswith(str(folder) + os.sep) and file_path != folder: return "Invalid path", 400 if not file_path.exists(): return "File not found", 404 return send_file(str(file_path), as_attachment=True) @app.route("/download-csv") @takeoff_workflow_required def download_csv(): folder_name = request.args.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return "Invalid folder", 400 folder = WATCH_DIR / folder_name state = _load_state(folder) csv_path = folder / state.get("csv_filename", "") if not csv_path.exists(): return "CSV not found", 404 return send_file(str(csv_path), as_attachment=True) @app.route("/list-shared-folders") @takeoff_workflow_required def list_shared_folders(): """Return sorted list of subdirectory names in SHARED_DIR.""" if not SHARED_DIR.is_dir(): return jsonify({"folders": []}) folders = sorted( (f.name for f in SHARED_DIR.iterdir() if f.is_dir() and not f.name.startswith(".")), reverse=True, ) return jsonify({"folders": folders}) @app.route("/select-shared-folder", methods=["POST"]) @takeoff_workflow_required def select_shared_folder(): """Load a shared folder: copy to WATCH_DIR if needed, parse CSV, auto-match files.""" data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 src = (SHARED_DIR / folder_name).resolve() if not str(src).startswith(str(SHARED_DIR.resolve())) or not src.is_dir(): return jsonify({"error": "Folder not found in shared directory"}), 404 # Use or create a working copy in WATCH_DIR dst = WATCH_DIR / folder_name if not dst.exists(): shutil.copytree(str(src), str(dst)) # Find CSV (root first, then subdirectories) csv_files = sorted(dst.glob("*.csv")) or sorted(dst.glob("**/*.csv")) if not csv_files: return jsonify({"error": "No CSV file found in folder"}), 400 csv_path = csv_files[0] # Parse same as upload-zip: col A=identifier, col C=item name, col H=ref items: list[str] = [] item_refs: list[str] = [] item_cols_a: list[str] = [] with open(csv_path, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) next(reader, None) for row in reader: if len(row) >= 3 and row[2].strip(): items.append(row[2].strip()) item_refs.append(row[7].strip() if len(row) >= 8 else "") item_cols_a.append(row[0].strip() if row else "") if not items: return jsonify({"error": "CSV has no items (expected data in column C from row 2 onward)"}), 400 auto_files = _match_files_to_items(dst, items, item_refs, item_cols_a) files_state = {str(i): dict(m) for i, m in enumerate(auto_files)} # Carry over carriers generated by a previous Submit All (state.json records # created=True + filename); their names don't follow the auto-match suffix # convention, so re-discovery relies on the prior state. try: prev_files = _load_state(dst).get("files", {}) except Exception: prev_files = {} if isinstance(prev_files, dict): for i in range(len(items)): pf = prev_files.get(str(i), {}) if (isinstance(pf, dict) and pf.get("created") and pf.get("carrier") and (dst / pf["carrier"]).is_file()): files_state[str(i)]["carrier"] = pf["carrier"] files_state[str(i)]["created"] = True auto_files[i]["carrier"] = pf["carrier"] state: dict = { "csv_filename": str(csv_path.relative_to(dst)), "stem": folder_name, "items": items, "item_refs": item_refs, "item_cols_a": item_cols_a, "files": files_state, "results": {}, } _save_state(dst, state) meta_path = dst / "project_metadata.json" has_metadata = meta_path.is_file() metadata = None if has_metadata: try: metadata = json.loads(meta_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): has_metadata = False return jsonify({ "folder": folder_name, "items": items, "item_refs": item_refs, "auto_files": auto_files, "has_metadata": has_metadata, "metadata": metadata, }) _META_OPTIONAL_KEYS = ("job_number", "production_date", "spec_type", "station") _META_OVERRIDE_KEYS = frozenset( ("customer_name", "project_name", "vendor_project_code") + _META_OPTIONAL_KEYS ) @app.route("/save-project-metadata", methods=["POST"]) @takeoff_workflow_required def save_project_metadata(): """Save project_metadata.json into the working folder in WATCH_DIR. Merges into any existing metadata file so callers that omit the optional keys (job_number, production_date, spec_type, station, item_overrides) leave previously saved values untouched. item_overrides, when present, replaces the stored per-item override map wholesale. """ data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 folder_path = (WATCH_DIR / folder_name).resolve() if not str(folder_path).startswith(str(WATCH_DIR.resolve())) or not folder_path.is_dir(): return jsonify({"error": "Folder not found"}), 404 meta_path = folder_path / "project_metadata.json" metadata: dict = {} if meta_path.is_file(): try: existing = json.loads(meta_path.read_text(encoding="utf-8")) if isinstance(existing, dict): metadata = existing except (json.JSONDecodeError, OSError): metadata = {} metadata.update({ "customer_name": str(data.get("customer_name") or "").strip(), "project_name": str(data.get("project_name") or "").strip(), "vendor_project_code": str(data.get("vendor_project_code") or "").strip(), "folder_name": folder_name, }) for key in _META_OPTIONAL_KEYS: if key in data: metadata[key] = str(data.get(key) or "").strip() item_overrides = data.get("item_overrides") if isinstance(item_overrides, dict): metadata["item_overrides"] = { str(idx): {k: str(v).strip() for k, v in fields.items() if k in _META_OVERRIDE_KEYS} for idx, fields in item_overrides.items() if isinstance(fields, dict) } metadata.setdefault("created_at", _datetime_type.now().isoformat()) metadata["updated_at"] = _datetime_type.now().isoformat() try: with open(meta_path, "w", encoding="utf-8") as fh: json.dump(metadata, fh, indent=2) except OSError as exc: return jsonify({"error": f"Failed to save metadata: {exc}"}), 500 return jsonify({"ok": True, "metadata": metadata}) @app.route("/preview-file") @takeoff_workflow_required def preview_file(): """Serve a job file inline (no download prompt) for PDF preview.""" folder_name = request.args.get("folder", "") filename = request.args.get("file", "") if not folder_name or not filename: return "Missing parameters", 400 if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return "Invalid folder", 400 folder = (WATCH_DIR / folder_name).resolve() file_path = (folder / filename).resolve() if not str(file_path).startswith(str(folder) + os.sep) and file_path != folder: return "Invalid path", 400 if not file_path.exists(): return "File not found", 404 return send_file(str(file_path), mimetype="application/pdf") # inline, no as_attachment @app.route("/takeoff/project-files") @takeoff_workflow_required def takeoff_project_files(): """List the files in a Take-off working folder for the Project Information popup's file browser/preview panels (DVI-641). Returns a flat, recursive listing so every project document is reachable in one list.""" folder_name = request.args.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify(error="Invalid folder"), 400 folder = (WATCH_DIR / folder_name).resolve() if not folder.is_dir(): return jsonify(error="Folder not found"), 404 skip_names = {"state.json"} files = [] for p in sorted(folder.rglob("*"), key=lambda x: str(x).lower()): if not p.is_file() or p.name.startswith(".") or p.name in skip_names: continue rel = p.relative_to(folder) parent = str(rel.parent).replace(os.sep, "/") files.append({ "name": p.name, "rel": str(rel).replace(os.sep, "/"), "dir": "" if parent == "." else parent, "ext": p.suffix.lower().lstrip("."), "size": p.stat().st_size, }) return jsonify(folder=folder_name, files=files) @app.route("/takeoff/file-preview") @takeoff_workflow_required def takeoff_file_preview(): """Serve a Take-off working-folder file inline for the Project Information popup preview pane (DVI-641). Unlike /preview-file (PDF-only), the mimetype is auto-detected so images and text files render inline too.""" folder_name = request.args.get("folder", "") rel = request.args.get("file", "") if not folder_name or not rel: return "Missing parameters", 400 if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name or ".." in rel: return "Invalid path", 400 folder = (WATCH_DIR / folder_name).resolve() file_path = (folder / rel).resolve() if not str(file_path).startswith(str(folder) + os.sep): return "Invalid path", 400 if not file_path.is_file(): return "File not found", 404 return send_file(str(file_path), download_name=file_path.name) # inline, auto mimetype @app.route("/download-job-zip") @takeoff_workflow_required def download_job_zip(): """Zip the entire job folder (excluding state.json) and stream it as a download.""" folder_name = request.args.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return "Invalid folder", 400 folder = (WATCH_DIR / folder_name).resolve() if not folder.is_dir(): return "Folder not found", 404 buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: for f in sorted(folder.rglob("*")): if f.is_file() and f.name != "state.json": zf.write(f, f.relative_to(folder)) buf.seek(0) return send_file( buf, as_attachment=True, download_name=f"{folder_name}.zip", mimetype="application/zip", ) @app.route("/publish", methods=["POST"]) @takeoff_workflow_required def publish(): """Copy job folder to SHARED_DIR. Errors if destination already exists.""" data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") overwrite = bool(data.get("overwrite", False)) if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 src = (WATCH_DIR / folder_name).resolve() if not src.is_dir(): return jsonify({"error": "Source folder not found"}), 400 SHARED_DIR.mkdir(parents=True, exist_ok=True) dst = (SHARED_DIR / folder_name).resolve() if not str(dst).startswith(str(SHARED_DIR.resolve())): return jsonify({"error": "Invalid destination path"}), 400 if dst.exists(): if not overwrite: return jsonify({"error": f'A folder named "{folder_name}" already exists in the shared directory', "exists": True}), 409 shutil.rmtree(str(dst)) shutil.copytree(str(src), str(dst)) return jsonify({"ok": True}) @app.route("/duplicate-folder", methods=["POST"]) @takeoff_workflow_required def duplicate_folder(): """Copy a job folder (documents, CSV, metadata, and all changes) to a new folder in WATCH_DIR. The copy is made as-is; no processing is applied.""" data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") new_name = (data.get("new_folder", "") or "").strip() if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid source folder"}), 400 if not _SAFE_JOB_RE.match(new_name) or ".." in new_name: return jsonify({"error": "Invalid folder name. Use letters, numbers, spaces, dashes, underscores or dots."}), 400 src = (WATCH_DIR / folder_name).resolve() if not src.is_dir(): return jsonify({"error": "Source folder not found"}), 400 dst = (WATCH_DIR / new_name).resolve() if not str(dst).startswith(str(WATCH_DIR.resolve())): return jsonify({"error": "Invalid destination path"}), 400 if dst.exists(): return jsonify({"error": f'A folder named "{new_name}" already exists.'}), 409 shutil.copytree(str(src), str(dst)) return jsonify({"ok": True, "folder": new_name}) # --------------------------------------------------------------------------- # Routes — Production Run (DVI-215) # --------------------------------------------------------------------------- PRODRUN_DIR = Path(os.environ.get("PRODRUN_DIR", "/var/www/html/prodrunmerge")) _SP_HOSTNAME = "icastinc.sharepoint.com" _SP_SITE_PATH = "/sites/StackIT" _SP_SYNC_PREFIX = "/sites/StackIT/Shared Documents/StackIT-to-Netsuite/" _SP_DRIVE_SYNC_ROOT = "StackIT-to-Netsuite" def _extract_pdf_url(viewer_url: str) -> str: """Extract direct PDF URL from Google Docs viewer wrapper URLs. Input: https://docs.google.com/viewer?url=http://fileshare.icastinc.com/shared/... Output: http://fileshare.icastinc.com/shared/... (with double-encoding resolved) Non-viewer URLs are returned unchanged. """ from urllib.parse import urlparse, parse_qs if "docs.google.com/viewer" in viewer_url: parsed = urlparse(viewer_url) qs = parse_qs(parsed.query) inner = qs.get("url", [""])[0] if inner: return inner # parse_qs already decodes %2520 → %20 return viewer_url def _resolve_sharepoint_url(url: str) -> dict | None: """Recognise SharePoint URLs from icastinc.sharepoint.com. Returns a typed dict: - AllItems.aspx?id= URLs → {type:"path", relative_path, fileshare_url, sp_server_path} - Sharing links (/:b:/s/…) → {type:"sharing_link", url} - Other SP URLs → {type:"sharing_link", url} (best-effort) - Non-SharePoint URLs → None """ from urllib.parse import urlparse, parse_qs, quote parsed = urlparse(url) if parsed.hostname != _SP_HOSTNAME: return None qs = parse_qs(parsed.query) id_param = qs.get("id", [""])[0] if id_param and id_param.startswith(_SP_SYNC_PREFIX): relative_path = id_param[len(_SP_SYNC_PREFIX):] if relative_path: _url_prefix = _load_shared_url_prefix() fileshare_url = f"{_url_prefix}/shared/{quote(relative_path, safe='/')}" return { "type": "path", "relative_path": relative_path, "fileshare_url": fileshare_url, "sp_server_path": id_param, } return {"type": "sharing_link", "url": url} def _download_from_sharepoint(sp_server_path: str, dest_dir: Path, index: int) -> Path | None: """Download a file directly from SharePoint using app-only credentials.""" try: from urllib.parse import quote as _q app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=[f"https://{_SP_HOSTNAME}/.default"]) if "access_token" not in token_result: print( f"[PRODRUN-SP] Failed to acquire SharePoint token: " f"{token_result.get('error_description', 'unknown')}", flush=True, ) return None encoded_path = _q(sp_server_path, safe="/") api_url = ( f"https://{_SP_HOSTNAME}{_SP_SITE_PATH}/_api/web/" f"GetFileByServerRelativeUrl('{encoded_path}')/$value" ) resp = _http_requests.get( api_url, headers={ "Authorization": f"Bearer {token_result['access_token']}", "Accept": "application/octet-stream", }, timeout=120, stream=True, ) resp.raise_for_status() fname = Path(sp_server_path).name if not fname or not fname.lower().endswith(".pdf"): fname = f"file_{index:04d}.pdf" fname = secure_filename(fname) dest = dest_dir / fname counter = 1 while dest.exists(): stem = dest.stem dest = dest_dir / f"{stem}_{counter}.pdf" counter += 1 with open(dest, "wb") as f: for chunk in resp.iter_content(8192): f.write(chunk) print(f"[PRODRUN-SP] Downloaded from SharePoint API: {dest.name}", flush=True) return dest except Exception as exc: print(f"[PRODRUN-SP] SharePoint API download failed: {exc}", flush=True) return None def _get_rclone_sp_token() -> str | None: """Get a Graph-compatible access token from rclone's OneDrive/SharePoint config. The rclone remote iCast_StackIT has delegated tokens with Files.Read.All and Sites.Read.All — sufficient for resolving sharing links. """ import subprocess from datetime import datetime, timezone try: output = subprocess.check_output( ["rclone", "config", "show", "iCast_StackIT"], text=True, timeout=10) except Exception as exc: print(f"[PRODRUN-SP] rclone config read failed: {exc}", flush=True) return None token_data = None for line in output.splitlines(): line = line.strip() if line.startswith("token ="): token_data = json.loads(line.split("=", 1)[1].strip()) break if not token_data or not token_data.get("access_token"): return None expiry = token_data.get("expiry", "") if expiry: try: exp_str = expiry.rstrip("Z") + "+00:00" if "+" not in expiry else expiry exp_dt = datetime.fromisoformat(exp_str) if exp_dt <= datetime.now(timezone.utc): import subprocess as _sp print("[PRODRUN-SP] rclone token expired, refreshing…", flush=True) _sp.run(["rclone", "lsf", "iCast_StackIT:", "--max-depth", "0"], timeout=30, capture_output=True) out2 = _sp.check_output( ["rclone", "config", "show", "iCast_StackIT"], text=True, timeout=10) for l2 in out2.splitlines(): l2 = l2.strip() if l2.startswith("token ="): token_data = json.loads(l2.split("=", 1)[1].strip()) break except Exception as exc: print(f"[PRODRUN-SP] rclone token refresh failed: {exc}", flush=True) return token_data.get("access_token") def _resolve_sp_sharing_link(url: str, dest_dir: Path, index: int) -> Path | None: """Resolve a SharePoint sharing link via Graph API. 1. GET /shares/{token}/driveItem → file name + parent path → local SHARED_DIR lookup 2. Fall back to fileshare URL download, then Graph direct download. Token sources: app credentials first, then rclone delegated token. """ sharing_token = "u!" + base64.urlsafe_b64encode(url.encode()).rstrip(b"=").decode() def _try_graph_token() -> str | None: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) return result.get("access_token") access_token = _try_graph_token() headers = {"Authorization": f"Bearer {access_token}"} if access_token else {} meta_resp = None if access_token: meta_resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/shares/{sharing_token}/driveItem", headers=headers, params={"$select": "name,parentReference,size"}, timeout=30, ) if meta_resp.status_code in (401, 403): print("[PRODRUN-SP] App credentials lack Graph file permissions, trying rclone token", flush=True) meta_resp = None if meta_resp is None or meta_resp.status_code in (401, 403): rclone_token = _get_rclone_sp_token() if rclone_token: access_token = rclone_token headers = {"Authorization": f"Bearer {access_token}"} meta_resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/shares/{sharing_token}/driveItem", headers=headers, params={"$select": "name,parentReference,size"}, timeout=30, ) else: print("[PRODRUN-SP] No rclone token available either", flush=True) return None try: fname = "" if meta_resp and meta_resp.status_code == 200: meta = meta_resp.json() fname = meta.get("name", "") parent_path = meta.get("parentReference", {}).get("path", "") print(f"[PRODRUN-SP] Sharing link resolved: name={fname!r} parent={parent_path!r}", flush=True) marker = f"{_SP_DRIVE_SYNC_ROOT}/" idx = parent_path.find(marker) if idx >= 0 and fname: rel_parent = parent_path[idx + len(marker):] local_path = SHARED_DIR / rel_parent / fname if local_path.exists(): print(f"[PRODRUN-SP] Sharing link → local file: {local_path}", flush=True) return local_path print(f"[PRODRUN-SP] Local not found: {local_path}", flush=True) from urllib.parse import quote _url_prefix = _load_shared_url_prefix() fileshare_url = f"{_url_prefix}/shared/{quote(rel_parent + '/' + fname, safe='/')}" print(f"[PRODRUN-SP] Trying fileshare URL: {fileshare_url}", flush=True) try: fs_resp = _http_requests.get(fileshare_url, timeout=120, stream=True) fs_resp.raise_for_status() dl_fname = secure_filename(fname) if fname else f"file_{index:04d}.pdf" dest = dest_dir / dl_fname counter = 1 while dest.exists(): dest = dest_dir / f"{dest.stem}_{counter}.pdf" counter += 1 with open(dest, "wb") as f: for chunk in fs_resp.iter_content(8192): f.write(chunk) print(f"[PRODRUN-SP] Downloaded via fileshare: {dest.name}", flush=True) return dest except Exception: print("[PRODRUN-SP] Fileshare download failed, trying Graph download", flush=True) else: status = meta_resp.status_code if meta_resp else "no response" body = meta_resp.text[:200] if meta_resp else "" print(f"[PRODRUN-SP] Graph driveItem lookup: {status} {body}", flush=True) # Download directly via Graph sharing API dl_resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/shares/{sharing_token}/driveItem/content", headers=headers, timeout=120, stream=True, ) dl_resp.raise_for_status() if not fname: cd = dl_resp.headers.get("Content-Disposition", "") if "filename=" in cd: fname = cd.split("filename=")[-1].strip('" ') if not fname or not fname.lower().endswith(".pdf"): fname = f"file_{index:04d}.pdf" fname = secure_filename(fname) dest = dest_dir / fname counter = 1 while dest.exists(): dest = dest_dir / f"{dest.stem}_{counter}.pdf" counter += 1 with open(dest, "wb") as f: for chunk in dl_resp.iter_content(8192): f.write(chunk) print(f"[PRODRUN-SP] Downloaded via Graph sharing API: {dest.name}", flush=True) return dest except Exception as exc: print(f"[PRODRUN-SP] Sharing link resolution failed: {exc}", flush=True) return None def _prodrun_merge_worker(job_id: str) -> None: """Background worker: download PDFs from saved state and merge each set.""" from datetime import date from urllib.parse import urlparse job_dir = PRODRUN_DIR / job_id state = _prodrun_load_state(job_dir) if not state: _prodrun_write_status(job_dir, {"status": "error", "error": "No state found"}) return date_str = date.today().strftime("%Y-%m-%d") date_dir = job_dir / date_str drawings_dir = date_dir / "drawings" labels_dir = date_dir / "labels" for d in (drawings_dir, labels_dir): d.mkdir(parents=True, exist_ok=True) _prodrun_write_status(job_dir, {"status": "processing", "message": "Preparing downloads…"}) items = state.get("items", []) overrides = state.get("overrides", {}) quantities = state.get("quantities", {}) # Build per-item source info, tracking item index for failure reporting drawing_sources: list[dict] = [] # {url|file, item_idx, doc_num, original_url} label_sources: list[dict] = [] skipped = 0 for i, item in enumerate(items): ov = overrides.get(str(i), {}) doc_num = item.get("doc_num", f"Row {i + 2}") # Drawing source draw_ov = ov.get("drawing") draw_src: dict | None = None if draw_ov and draw_ov.get("type") == "removed": pass elif draw_ov and draw_ov.get("type") == "file": draw_src = {"file": draw_ov["value"], "job_dir": str(job_dir)} elif draw_ov and draw_ov.get("type") == "url" and draw_ov.get("value"): draw_src = {"url": _extract_pdf_url(draw_ov["value"]), "original_url": draw_ov["value"]} elif item.get("drawing_url"): draw_src = {"url": _extract_pdf_url(item["drawing_url"]), "original_url": item["drawing_url"]} if draw_src: draw_src["item_idx"] = i draw_src["doc_num"] = doc_num draw_src["row_num"] = item.get("row_num", i + 2) draw_src["qty"] = int(quantities.get(str(i), 1)) drawing_sources.append(draw_src) # Label source lbl_ov = ov.get("label") lbl_src: dict | None = None if lbl_ov and lbl_ov.get("type") == "removed": pass elif lbl_ov and lbl_ov.get("type") == "file": lbl_src = {"file": lbl_ov["value"], "job_dir": str(job_dir)} elif lbl_ov and lbl_ov.get("type") == "url" and lbl_ov.get("value"): lbl_src = {"url": _extract_pdf_url(lbl_ov["value"]), "original_url": lbl_ov["value"]} elif item.get("label_url"): lbl_src = {"url": _extract_pdf_url(item["label_url"]), "original_url": item["label_url"]} if lbl_src: lbl_src["item_idx"] = i lbl_src["doc_num"] = doc_num lbl_src["row_num"] = item.get("row_num", i + 2) lbl_src["qty"] = int(quantities.get(str(i), 1)) label_sources.append(lbl_src) if not item.get("drawing_url") and not item.get("label_url") and str(i) not in overrides: skipped += 1 _prodrun_write_status(job_dir, {"status": "processing", "message": f"Downloading {len(drawing_sources)} drawings and {len(label_sources)} labels…"}) def _download_pdf(url: str, dest_dir: Path, index: int) -> Path | None: try: resp = _http_requests.get(url, timeout=120, stream=True) resp.raise_for_status() parsed = urlparse(url) fname = Path(parsed.path).name if parsed.path else "" if not fname or not fname.lower().endswith(".pdf"): fname = f"file_{index:04d}.pdf" fname = secure_filename(fname) dest = dest_dir / fname counter = 1 while dest.exists(): stem = dest.stem dest = dest_dir / f"{stem}_{counter}.pdf" counter += 1 with open(dest, "wb") as f: for chunk in resp.iter_content(8192): f.write(chunk) return dest except Exception: return None def _resolve_source(src: dict, dest_dir: Path, index: int) -> Path | None: if "file" in src: p = Path(src["job_dir"]) / src["file"] if p.exists() and p.suffix.lower() == ".pdf": return p return None url = src["url"] sp_info = _resolve_sharepoint_url(url) if sp_info: if sp_info["type"] == "sharing_link": return _resolve_sp_sharing_link(sp_info["url"], dest_dir, index) local_path = SHARED_DIR / sp_info["relative_path"] if local_path.exists() and local_path.suffix.lower() == ".pdf": print(f"[PRODRUN-SP] Using local synced file: {local_path}", flush=True) return local_path print(f"[PRODRUN-SP] Local not found, trying fileshare URL", flush=True) result = _download_pdf(sp_info["fileshare_url"], dest_dir, index) if result: return result print(f"[PRODRUN-SP] Fileshare failed, trying SharePoint API", flush=True) return _download_from_sharepoint(sp_info["sp_server_path"], dest_dir, index) return _download_pdf(url, dest_dir, index) drawing_files: list[tuple[Path, int]] = [] # (path, quantity) failed_items: list[dict] = [] # [{item_idx, doc_num, type, url}] for i, src in enumerate(drawing_sources): p = _resolve_source(src, drawings_dir, i) if p: drawing_files.append((p, src.get("qty", 1))) else: failed_items.append({ "item_idx": src["item_idx"], "doc_num": src["doc_num"], "row_num": src.get("row_num"), "type": "drawing", "url": src.get("original_url", src.get("url", src.get("file", ""))), }) label_files: list[tuple[Path, int]] = [] # (path, quantity) for i, src in enumerate(label_sources): p = _resolve_source(src, labels_dir, i) if p: label_files.append((p, src.get("qty", 1))) else: failed_items.append({ "item_idx": src["item_idx"], "doc_num": src["doc_num"], "row_num": src.get("row_num"), "type": "label", "url": src.get("original_url", src.get("url", src.get("file", ""))), }) _prodrun_write_status(job_dir, {"status": "processing", "message": f"Merging {len(drawing_files)} drawings and {len(label_files)} labels…"}) from pypdf import PdfWriter result: dict = { "status": "done", "skipped_rows": skipped, "failed_downloads": len(failed_items), "failed_items": failed_items, } if drawing_files: try: writer = PdfWriter() for pdf_path, qty in drawing_files: for _ in range(qty): writer.append(str(pdf_path)) merged_name = f"{date_str} Drawing Merge.pdf" merged_path = date_dir / merged_name with open(merged_path, "wb") as f: writer.write(f) result["drawing_merge"] = { "filename": merged_name, "job_id": job_id, "page_count": len(writer.pages), "preview_url": f"prodrun-preview?job_id={quote(job_id, safe='')}&file={quote(merged_name, safe='')}", "download_url": f"prodrun-download?job_id={quote(job_id, safe='')}&file={quote(merged_name, safe='')}", } except Exception as exc: result["drawing_merge_error"] = str(exc) if label_files: try: writer = PdfWriter() for pdf_path, qty in label_files: for _ in range(qty): writer.append(str(pdf_path)) merged_name = f"{date_str} Label Merge.pdf" merged_path = date_dir / merged_name with open(merged_path, "wb") as f: writer.write(f) result["label_merge"] = { "filename": merged_name, "job_id": job_id, "page_count": len(writer.pages), "preview_url": f"prodrun-preview?job_id={quote(job_id, safe='')}&file={quote(merged_name, safe='')}", "download_url": f"prodrun-download?job_id={quote(job_id, safe='')}&file={quote(merged_name, safe='')}", } except Exception as exc: result["label_merge_error"] = str(exc) _prodrun_write_status(job_dir, result) def _prodrun_find_file(job_id: str, filename: str) -> Path | None: """Locate a merged PDF for a job, regardless of which date subdir it sits in. Merged files are written under PRODRUN_DIR///. Live runs use today's date, but History opens older runs, so we search all date subdirs instead of assuming date.today(). """ base = (PRODRUN_DIR / job_id).resolve() if not str(base).startswith(str(PRODRUN_DIR.resolve())): return None if not base.exists(): return None for sub in base.iterdir(): if sub.is_dir(): cand = sub / filename if cand.exists(): return cand return None def _prodrun_state_path(job_dir: Path) -> Path: return job_dir / "prodrun_state.json" def _prodrun_load_state(job_dir: Path) -> dict: p = _prodrun_state_path(job_dir) return json.loads(p.read_text()) if p.exists() else {} def _prodrun_save_state(job_dir: Path, state: dict) -> None: _prodrun_state_path(job_dir).write_text(json.dumps(state, indent=2)) def _prodrun_write_status(job_dir: Path, status: dict) -> None: (job_dir / "status.json").write_text(json.dumps(status, indent=2)) def _parse_csv_prodrun(csv_path: Path) -> list[dict]: items: list[dict] = [] with open(csv_path, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) next(reader, None) # skip header for row_idx, row in enumerate(reader, start=2): doc_num = row[1].strip() if len(row) > 1 else "" customer = row[7].strip() if len(row) > 7 else "" so_num = row[8].strip() if len(row) > 8 else "" col_o = row[14].strip() if len(row) > 14 else "" col_p = row[15].strip() if len(row) > 15 else "" if not doc_num and not col_o and not col_p: continue items.append({ "row_num": row_idx, "doc_num": doc_num, "customer": customer, "so_num": so_num, "drawing_url": col_o, "label_url": col_p, }) return items def _parse_xlsx_prodrun(xlsx_path: Path) -> list[dict]: wb = load_workbook(xlsx_path, read_only=True, data_only=True) # Use the second worksheet which contains the raw data ws = wb.worksheets[1] if len(wb.worksheets) > 1 else wb.worksheets[0] items: list[dict] = [] for row_idx, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2): if not row or len(row) < 2: continue doc_num = str(row[1]).strip() if row[1] is not None else "" customer = str(row[7]).strip() if len(row) > 7 and row[7] is not None else "" so_num = str(row[8]).strip() if len(row) > 8 and row[8] is not None else "" col_o = str(row[14]).strip() if len(row) > 14 and row[14] is not None else "" col_p = str(row[15]).strip() if len(row) > 15 and row[15] is not None else "" if not doc_num and not col_o and not col_p: continue items.append({ "row_num": row_idx, "doc_num": doc_num, "customer": customer, "so_num": so_num, "drawing_url": col_o, "label_url": col_p, }) wb.close() return items @app.route("/prodrun-upload", methods=["POST"]) @prod_run_required def prodrun_upload(): """Accept a CSV or XLSX, parse items with Document Numbers and URLs, return for review.""" csv_file = request.files.get("csv") if not csv_file or not csv_file.filename: return jsonify({"error": "No file provided"}), 400 fname_lower = csv_file.filename.lower() if not fname_lower.endswith(".csv") and not fname_lower.endswith(".xlsx"): return jsonify({"error": "File must be a .csv or .xlsx"}), 400 job_id = str(uuid.uuid4())[:8] job_dir = PRODRUN_DIR / job_id job_dir.mkdir(parents=True, exist_ok=True) saved_path = job_dir / secure_filename(csv_file.filename) csv_file.save(saved_path) # Parse rows: col B = Document Number, col H = Customer, col I = SO#, # col O = Drawing URL, col P = Label URL items: list[dict] = [] if fname_lower.endswith(".xlsx"): items = _parse_xlsx_prodrun(saved_path) else: items = _parse_csv_prodrun(saved_path) if not items: shutil.rmtree(str(job_dir), ignore_errors=True) return jsonify({"error": "No items found in file"}), 400 _prodrun_save_state(job_dir, { "csv_filename": secure_filename(csv_file.filename), "items": items, "overrides": {}, }) return jsonify({"folder": job_id, "items": items}) @app.route("/prodrun-upload-file", methods=["POST"]) @prod_run_required def prodrun_upload_file(): """Accept a per-item file override for production run merge.""" f = request.files.get("file") folder = request.form.get("folder", "") item_index = request.form.get("item_index", "") file_type = request.form.get("file_type", "") if not f or not f.filename or not folder or not file_type: return jsonify({"error": "Missing parameters"}), 400 if ".." in folder: return jsonify({"error": "Invalid folder"}), 400 if not f.filename.lower().endswith(".pdf"): return jsonify({"error": "File must be a PDF"}), 400 job_dir = (PRODRUN_DIR / folder).resolve() if not str(job_dir).startswith(str(PRODRUN_DIR.resolve())): return jsonify({"error": "Invalid folder"}), 400 job_dir.mkdir(parents=True, exist_ok=True) fname = secure_filename(f.filename) f.save(job_dir / fname) return jsonify({"filename": fname}) @app.route("/prodrun-submit", methods=["POST"]) @prod_run_required def prodrun_submit(): """Accept reviewed items with overrides and start the merge.""" data = request.get_json(silent=True) or {} folder = data.get("folder", "") overrides = data.get("overrides", {}) quantities = data.get("quantities", {}) if not folder or ".." in folder: return jsonify({"error": "Invalid folder"}), 400 job_dir = (PRODRUN_DIR / folder).resolve() if not str(job_dir).startswith(str(PRODRUN_DIR.resolve())): return jsonify({"error": "Invalid folder"}), 400 state = _prodrun_load_state(job_dir) if not state: return jsonify({"error": "No upload state found — upload CSV first"}), 400 state["overrides"] = overrides state["quantities"] = quantities _prodrun_save_state(job_dir, state) t = threading.Thread(target=_prodrun_merge_worker, args=(folder,), daemon=True) t.start() return jsonify({"job_id": folder}) @app.route("/prodrun-status") @prod_run_required def prodrun_status(): """Poll for production run merge status.""" job_id = request.args.get("job_id", "") if not job_id or ".." in job_id: return jsonify({"error": "Invalid job_id"}), 400 status_path = PRODRUN_DIR / job_id / "status.json" if not status_path.exists(): return jsonify({"status": "pending", "message": "Initializing…"}) try: return jsonify(json.loads(status_path.read_text())) except Exception: return jsonify({"status": "pending", "message": "Reading status…"}) @app.route("/prodrun-preview") @prod_run_required def prodrun_preview(): """Serve a merged production run PDF inline for preview.""" job_id = request.args.get("job_id", "") filename = request.args.get("file", "") if not job_id or not filename or ".." in job_id or ".." in filename: return "Invalid parameters", 400 file_path = _prodrun_find_file(job_id, filename) if not file_path: return "File not found", 404 return send_file(str(file_path), mimetype="application/pdf") @app.route("/prodrun-download") @prod_run_required def prodrun_download(): """Download a merged production run PDF.""" job_id = request.args.get("job_id", "") filename = request.args.get("file", "") if not job_id or not filename or ".." in job_id or ".." in filename: return "Invalid parameters", 400 file_path = _prodrun_find_file(job_id, filename) if not file_path: return "File not found", 404 return send_file(str(file_path), as_attachment=True, download_name=filename) @app.route("/prodrun-history") @prod_run_required def prodrun_history(): """List completed production runs, latest first (DVI-528). Each entry summarizes a finished merge job; the client reuses /prodrun-status?job_id= to load full detail (merged-file preview / download URLs) when an entry is opened. """ runs: list[dict] = [] if PRODRUN_DIR.exists(): for job_dir in PRODRUN_DIR.iterdir(): if not job_dir.is_dir(): continue status_path = job_dir / "status.json" if not status_path.exists(): continue try: status = json.loads(status_path.read_text()) except Exception: continue if status.get("status") != "done": continue state = _prodrun_load_state(job_dir) items = state.get("items", []) customer = "" so_num = "" for it in items: if not customer and it.get("customer"): customer = it["customer"] if not so_num and it.get("so_num"): so_num = it["so_num"] if customer and so_num: break dm = status.get("drawing_merge") or {} lm = status.get("label_merge") or {} try: mtime = status_path.stat().st_mtime except OSError: mtime = 0 runs.append({ "job_id": job_dir.name, "timestamp_ms": int(mtime * 1000), "csv_filename": state.get("csv_filename", ""), "item_count": len(items), "customer": customer, "so_num": so_num, "drawing_pages": dm.get("page_count"), "label_pages": lm.get("page_count"), "failed_downloads": status.get("failed_downloads", 0), }) runs.sort(key=lambda r: r["timestamp_ms"], reverse=True) return jsonify({"runs": runs}) # --------------------------------------------------------------------------- # Routes — Work Order Request (DVI-225) # --------------------------------------------------------------------------- def _generate_wor_pdf(data: dict, maintenance: dict | None = None) -> bytes: """Generate a Work Order Request PDF matching the ICAST template layout. *maintenance* — optional dict with keys ``accepted_by``, ``date``, ``work_order_number`` to fill the "For use by maintenance department" block. """ pdf = FPDF(orientation="P", unit="mm", format="letter") pdf.set_auto_page_break(auto=False) pdf.add_page() pw = pdf.w - 20 # page width minus margins (10mm each side) teal = (0, 159, 175) dark_gray = (80, 80, 80) white = (255, 255, 255) light_bg = (240, 250, 252) black = (0, 0, 0) # Title pdf.set_font("Helvetica", "B", 22) pdf.set_text_color(*black) pdf.cell(50, 12, "ICAST", ln=0) pdf.cell(0, 12, "Maintenance Work Order", ln=1) pdf.ln(4) def section_header(title): pdf.set_fill_color(*teal) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 10) pdf.cell(pw, 7, " " + title, fill=True, ln=1) pdf.set_text_color(*black) def label_value_row(pairs, row_h=7): col_w = pw / len(pairs) pdf.set_font("Helvetica", "B", 7) for label, _ in pairs: pdf.cell(col_w, 4, label) pdf.ln(4) pdf.set_font("Helvetica", "", 9) for _, val in pairs: pdf.cell(col_w, row_h, str(val or ""), border="B") pdf.ln(row_h) # Requestor Information section_header("Requestor information") label_value_row([ ("Requestor name", data.get("requestor_name", "")), ("Phone #", data.get("phone", "")), ("Email", data.get("email", "")), ]) pdf.ln(3) # Request Details section_header("Request details") label_value_row([ ("Date of request", data.get("date_of_request", "")), ("Location", data.get("location", "")), ]) label_value_row([ ("Building", data.get("building", "")), ("Man Hole", data.get("man_hole", "")), ("Equipment/asset/system affected", data.get("equipment", "")), ]) _manual = data.get("manual") or {} if _manual.get("original_name"): pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 4, "Attached repair / operator manual") pdf.ln(4) pdf.set_font("Helvetica", "", 9) pdf.cell(pw, 6, str(_manual.get("original_name", "")), border="B", ln=1) pdf.ln(3) # Request Description pdf.set_fill_color(*light_bg) pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Request description", ln=1) pdf.set_font("Helvetica", "", 9) desc = data.get("description", "") pdf.set_draw_color(180, 180, 180) x0, y0 = pdf.get_x(), pdf.get_y() pdf.rect(x0, y0, pw, 40) pdf.set_xy(x0 + 2, y0 + 1) pdf.multi_cell(pw - 4, 4.5, desc) pdf.set_y(y0 + 41) # Type of Issue pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Type of issue (check all that apply)", ln=1) issue_types = data.get("issue_type", []) if isinstance(issue_types, str): issue_types = [issue_types] other_text = data.get("issue_type_other", "") all_types = [ "Automotive", "Grounds", "Plumbing", "Custodial", "Cranes", "Concrete Forms", "Telehandler/Forklift", "Machine", "Electrical", "Tool Repair", "Safety", ] col_w = pw / 3 for row_start in range(0, len(all_types), 3): row_items = all_types[row_start:row_start + 3] for item in row_items: checked = item in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(col_w - 5, 5, " " + item) pdf.ln(5.5) # Other row other_checked = "Other" in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if other_checked else "", border=1) pdf.set_font("Helvetica", "", 8) other_display = "Other: " + other_text if other_checked and other_text else "Other:" pdf.cell(col_w - 5, 5, " " + other_display) pdf.ln(6) # Best times / Urgency / Desired completion (two-column layout) y_section = pdf.get_y() half = pw / 2 lm = pdf.l_margin # left margin x # -- Left column: best times + desired completion -- pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Best times for service", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("best_times", ""), border="B", ln=1) pdf.ln(2) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Desired completion date", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("desired_completion_date", ""), border="B", ln=1) left_end_y = pdf.get_y() # -- Right column: urgency checkboxes -- urgency = data.get("urgency", "") urg_x = lm + half pdf.set_xy(urg_x, y_section) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Urgency (check one)", ln=1) pdf.set_x(urg_x) for level in ["Low", "Medium", "High"]: checked = urgency == level pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5.5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(half - 5, 5.5, " " + level, ln=1) pdf.set_x(urg_x) right_end_y = pdf.get_y() pdf.set_y(max(left_end_y, right_end_y) + 4) pdf.set_fill_color(*dark_gray) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 9) pdf.cell(pw, 7, " For use by maintenance department", fill=True, ln=1) pdf.set_text_color(*black) m = maintenance or {} wo_num = m.get("work_order_number", "") or (f"{data['_wor_number']:05d}" if data.get("_wor_number") else "") label_value_row([ ("Accepted by", m.get("accepted_by", "")), ("Date", m.get("date", "")), ("Assigned work order #", wo_num), ], row_h=10) return pdf.output() def _generate_preventive_wor_pdf(data: dict, maintenance: dict | None = None) -> bytes: """Generate a Preventative Maintenance Work Order PDF. Mirrors the standard maintenance template plus a Parts table (matching the Mobile Machine layout). The "Completed / In progress" status selections are intentionally omitted — status is tracked by Work Order Admin processing. """ pdf = FPDF(orientation="P", unit="mm", format="letter") pdf.set_auto_page_break(auto=False) pdf.add_page() pw = pdf.w - 20 teal = (0, 159, 175) dark_gray = (80, 80, 80) white = (255, 255, 255) light_bg = (240, 250, 252) black = (0, 0, 0) # Title pdf.set_font("Helvetica", "B", 22) pdf.set_text_color(*black) pdf.cell(50, 12, "ICAST", ln=0) pdf.set_font("Helvetica", "B", 16) pdf.cell(0, 12, "Preventative Maintenance Work Order", ln=1) pdf.ln(4) def section_header(title): pdf.set_fill_color(*teal) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 10) pdf.cell(pw, 7, " " + title, fill=True, ln=1) pdf.set_text_color(*black) def label_value_row(pairs, row_h=7): col_w = pw / len(pairs) pdf.set_font("Helvetica", "B", 7) for label, _ in pairs: pdf.cell(col_w, 4, label) pdf.ln(4) pdf.set_font("Helvetica", "", 9) for _, val in pairs: pdf.cell(col_w, row_h, str(val or ""), border="B") pdf.ln(row_h) # Requestor Information section_header("Requestor information") label_value_row([ ("Requestor name", data.get("requestor_name", "")), ("Phone #", data.get("phone", "")), ("Email", data.get("email", "")), ]) pdf.ln(3) # Request Details section_header("Request details") label_value_row([ ("Date of request", data.get("date_of_request", "")), ("Location", data.get("location", "")), ]) label_value_row([ ("Building", data.get("building", "")), ("Man Hole", data.get("man_hole", "")), ("Equipment/asset/system affected", data.get("equipment", "")), ]) _manual = data.get("manual") or {} if _manual.get("original_name"): pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 4, "Attached repair / operator manual") pdf.ln(4) pdf.set_font("Helvetica", "", 9) pdf.cell(pw, 6, str(_manual.get("original_name", "")), border="B", ln=1) pdf.ln(3) # Request Description pdf.set_fill_color(*light_bg) pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Request description", ln=1) pdf.set_font("Helvetica", "", 9) desc = data.get("description", "") pdf.set_draw_color(180, 180, 180) x0, y0 = pdf.get_x(), pdf.get_y() pdf.rect(x0, y0, pw, 28) pdf.set_xy(x0 + 2, y0 + 1) pdf.multi_cell(pw - 4, 4.5, desc) pdf.set_y(y0 + 29) # Type of Issue pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Type of issue (check all that apply)", ln=1) issue_types = data.get("issue_type", []) if isinstance(issue_types, str): issue_types = [issue_types] other_text = data.get("issue_type_other", "") all_types = [ "Automotive", "Grounds", "Plumbing", "Custodial", "Cranes", "Concrete Forms", "Telehandler/Forklift", "Machine", "Electrical", "Tool Repair", "Safety", ] col_w = pw / 3 for row_start in range(0, len(all_types), 3): row_items = all_types[row_start:row_start + 3] for item in row_items: checked = item in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(col_w - 5, 5, " " + item) pdf.ln(5.5) other_checked = "Other" in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if other_checked else "", border=1) pdf.set_font("Helvetica", "", 8) other_display = "Other: " + other_text if other_checked and other_text else "Other:" pdf.cell(col_w - 5, 5, " " + other_display) pdf.ln(6) # Best times / Urgency / Desired completion (two-column layout) y_section = pdf.get_y() half = pw / 2 lm = pdf.l_margin pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Best times for service", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("best_times", ""), border="B", ln=1) pdf.ln(2) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Desired completion date", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("desired_completion_date", ""), border="B", ln=1) left_end_y = pdf.get_y() urgency = data.get("urgency", "") urg_x = lm + half pdf.set_xy(urg_x, y_section) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Urgency (check one)", ln=1) pdf.set_x(urg_x) for level in ["Low", "Medium", "High"]: checked = urgency == level pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5.5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(half - 5, 5.5, " " + level, ln=1) pdf.set_x(urg_x) right_end_y = pdf.get_y() pdf.set_y(max(left_end_y, right_end_y) + 4) # Parts table section_header("Parts") pdf.set_font("Helvetica", "B", 7) pdf.cell(pw * 0.2, 5, "Part Number") pdf.cell(pw * 0.4, 5, "Part Name") pdf.cell(pw * 0.2, 5, "Quantity") pdf.cell(pw * 0.2, 5, "Amount") pdf.ln(5) parts = data.get("parts", []) pdf.set_font("Helvetica", "", 8) for part in parts[:8]: pdf.cell(pw * 0.2, 6, str(part.get("number", "")), border="B") pdf.cell(pw * 0.4, 6, str(part.get("name", "")), border="B") pdf.cell(pw * 0.2, 6, str(part.get("qty", "")), border="B") pdf.cell(pw * 0.2, 6, str(part.get("amount", "")), border="B") pdf.ln(6) if not parts: for _ in range(3): pdf.cell(pw * 0.2, 6, "", border="B") pdf.cell(pw * 0.4, 6, "", border="B") pdf.cell(pw * 0.2, 6, "", border="B") pdf.cell(pw * 0.2, 6, "", border="B") pdf.ln(6) pdf.ln(1) pdf.set_font("Helvetica", "", 8) pdf.cell(pw, 5, "Parts Total: " + str(data.get("parts_total", "")), align="R") pdf.ln(7) # For use by maintenance department pdf.set_fill_color(*dark_gray) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 9) pdf.cell(pw, 7, " For use by maintenance department", fill=True, ln=1) pdf.set_text_color(*black) m = maintenance or {} wo_num = m.get("work_order_number", "") or (f"{data['_wor_number']:05d}" if data.get("_wor_number") else "") label_value_row([ ("Accepted by", m.get("accepted_by", "")), ("Date", m.get("date", "")), ("Assigned work order #", wo_num), ], row_h=10) return pdf.output() def _generate_mm_wor_pdf(data: dict) -> bytes: """Generate a Mobile Machine Work Order PDF matching the ICAST template.""" pdf = FPDF(orientation="P", unit="mm", format="letter") pdf.set_auto_page_break(auto=False) pdf.add_page() pw = pdf.w - 20 teal = (0, 159, 175) dark_gray = (80, 80, 80) white = (255, 255, 255) black = (0, 0, 0) light_bg = (240, 250, 252) # Title pdf.set_font("Helvetica", "B", 22) pdf.set_text_color(*black) pdf.cell(50, 12, "ICAST", ln=0) pdf.cell(0, 12, "Mobile Machine WORK ORDER", ln=1) pdf.ln(4) def section_header(title): pdf.set_fill_color(*teal) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 10) pdf.cell(pw, 7, " " + title, fill=True, ln=1) pdf.set_text_color(*black) def label_value_row(pairs, row_h=7): col_w = pw / len(pairs) pdf.set_font("Helvetica", "B", 7) for label, _ in pairs: pdf.cell(col_w, 4, label) pdf.ln(4) pdf.set_font("Helvetica", "", 9) for _, val in pairs: pdf.cell(col_w, row_h, str(val or ""), border="B") pdf.ln(row_h) # Machine Information section_header("Machine Information") label_value_row([ ("Machine", data.get("machine", "")), ("Order Number", data.get("_order_number", "") or (f"{data['_wor_number']:05d}" if data.get("_wor_number") else "")), ]) label_value_row([ ("Asset #", data.get("asset_number", "")), ("Order Received By", data.get("order_received_by", "")), ]) label_value_row([ ("Order Date & Time", data.get("order_date_time", "")), ("Date Promised", data.get("date_promised", "")), ("Date Delivered", data.get("date_delivered", "")), ]) pdf.ln(2) # Vehicle Information section_header("Vehicle Information") label_value_row([("VIN", data.get("vin", "")), ("Make and Model", data.get("make_model", ""))]) label_value_row([ ("Odometer Reading", data.get("odometer", "")), ("Motor #", data.get("motor_number", "")), ]) label_value_row([("License # and State", data.get("license_state", ""))]) pdf.ln(2) # Service Type checkboxes pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Service Type", ln=1) service_types = data.get("service_type", []) if isinstance(service_types, str): service_types = [service_types] all_services = [ "Lubricate", "Wiper Replacement", "Air Filter Replacement", "Change Oil", "Boom Repair", "Spark Plug Replacement", "Transmission Service", "Fuel System", "Exhaust System Check", "Battery Replacement / Check", "Tire Replacement", "Electronics", "Differential Service", "Brake Inspection / Replacement", "Suspension and Steering Check", "Flat Tire Repair", "Coolant System Flush", "Hydraulic repair / fill", ] col_w = pw / 3 for row_start in range(0, len(all_services), 3): row_items = all_services[row_start:row_start + 3] for item in row_items: checked = item in service_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 7) pdf.cell(col_w - 5, 5, " " + item) pdf.ln(5.5) pdf.ln(2) # Labor Description pdf.set_fill_color(*light_bg) pdf.set_font("Helvetica", "B", 7) labor_header_y = pdf.get_y() pdf.cell(pw - 30, 5, "Labor Description", ln=0) pdf.cell(30, 5, "Time", ln=1) pdf.set_font("Helvetica", "", 9) desc = data.get("labor_description", "") x0, y0 = pdf.get_x(), pdf.get_y() pdf.set_draw_color(180, 180, 180) pdf.rect(x0, y0, pw, 25) pdf.set_xy(x0 + 2, y0 + 1) pdf.multi_cell(pw - 4, 4.5, desc) pdf.set_y(y0 + 26) pdf.set_font("Helvetica", "B", 7) pdf.cell(pw / 2, 5, "", ln=0) pdf.cell(pw / 4, 5, "Time: " + str(data.get("labor_time", "")), ln=0) pdf.cell(pw / 4, 5, "Labor Total: " + str(data.get("labor_total", "")), ln=1) pdf.ln(2) # Parts table section_header("Parts") pdf.set_font("Helvetica", "B", 7) pdf.cell(pw * 0.2, 5, "Part Number") pdf.cell(pw * 0.4, 5, "Part Name") pdf.cell(pw * 0.2, 5, "Quantity") pdf.cell(pw * 0.2, 5, "Amount") pdf.ln(5) parts = data.get("parts", []) pdf.set_font("Helvetica", "", 8) for part in parts[:8]: pdf.cell(pw * 0.2, 6, str(part.get("number", "")), border="B") pdf.cell(pw * 0.4, 6, str(part.get("name", "")), border="B") pdf.cell(pw * 0.2, 6, str(part.get("qty", "")), border="B") pdf.cell(pw * 0.2, 6, str(part.get("amount", "")), border="B") pdf.ln(6) if not parts: for _ in range(3): pdf.cell(pw * 0.2, 6, "", border="B") pdf.cell(pw * 0.4, 6, "", border="B") pdf.cell(pw * 0.2, 6, "", border="B") pdf.cell(pw * 0.2, 6, "", border="B") pdf.ln(6) pdf.ln(1) pdf.set_font("Helvetica", "", 8) pdf.cell(pw, 5, "Parts Total: " + str(data.get("parts_total", "")), align="R") pdf.ln(7) # Authorization pdf.set_fill_color(*dark_gray) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 9) pdf.cell(pw, 7, " Authorization", fill=True, ln=1) pdf.set_text_color(*black) label_value_row([ ("Work Order Compiled By", data.get("compiled_by", "")), ("Date", data.get("compiled_date", "")), ]) label_value_row([ ("Work Authorized By", data.get("authorized_by", "")), ("Authorization Date", data.get("authorization_date", "")), ]) return pdf.output() def _generate_safety_wor_pdf(data: dict, safety: dict | None = None) -> bytes: """Generate a Safety Concerns Work Order PDF matching the ICAST template layout. *safety* — optional dict with keys ``accepted_by``, ``date``, ``work_order_number`` to fill the "For use by Safety department" block. Status checkboxes from the paper template (Completed, In progress/parts ordered) are intentionally omitted — those states are tracked by the Work Order Admin workflow. """ pdf = FPDF(orientation="P", unit="mm", format="letter") pdf.set_auto_page_break(auto=False) pdf.add_page() pw = pdf.w - 20 # page width minus margins (10mm each side) teal = (0, 159, 175) dark_gray = (80, 80, 80) white = (255, 255, 255) light_bg = (240, 250, 252) black = (0, 0, 0) # Title pdf.set_font("Helvetica", "B", 22) pdf.set_text_color(*black) pdf.cell(50, 12, "ICAST", ln=0) pdf.cell(0, 12, "Safety Concerns", ln=1) pdf.ln(4) def section_header(title): pdf.set_fill_color(*teal) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 10) pdf.cell(pw, 7, " " + title, fill=True, ln=1) pdf.set_text_color(*black) def label_value_row(pairs, row_h=7): col_w = pw / len(pairs) pdf.set_font("Helvetica", "B", 7) for label, _ in pairs: pdf.cell(col_w, 4, label) pdf.ln(4) pdf.set_font("Helvetica", "", 9) for _, val in pairs: pdf.cell(col_w, row_h, str(val or ""), border="B") pdf.ln(row_h) # Requestor Information section_header("Requestor information") label_value_row([ ("Requestor name", data.get("requestor_name", "")), ("Phone #", data.get("phone", "")), ("Email", data.get("email", "")), ]) pdf.ln(3) # Request Details section_header("Request details") label_value_row([ ("Date of request", data.get("date_of_request", "")), ("Location", data.get("location", "")), ]) label_value_row([ ("Building", data.get("building", "")), ("Man Hole", data.get("man_hole", "")), ("Equipment/asset/system affected", data.get("equipment", "")), ]) _manual = data.get("manual") or {} if _manual.get("original_name"): pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 4, "Attached repair / operator manual") pdf.ln(4) pdf.set_font("Helvetica", "", 9) pdf.cell(pw, 6, str(_manual.get("original_name", "")), border="B", ln=1) pdf.ln(3) # Request Description pdf.set_fill_color(*light_bg) pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Request description", ln=1) pdf.set_font("Helvetica", "", 9) desc = data.get("description", "") pdf.set_draw_color(180, 180, 180) x0, y0 = pdf.get_x(), pdf.get_y() pdf.rect(x0, y0, pw, 40) pdf.set_xy(x0 + 2, y0 + 1) pdf.multi_cell(pw - 4, 4.5, desc) pdf.set_y(y0 + 41) # Type of Safety Issue pdf.set_font("Helvetica", "B", 7) pdf.cell(pw, 5, "Type of Safety issue (check all that apply)", ln=1) issue_types = data.get("issue_type", []) if isinstance(issue_types, str): issue_types = [issue_types] other_text = data.get("issue_type_other", "") all_types = [ "Slip/Trip/Fall", "Isleway or Egress", "General Safety", "Lighting", "Near Miss", "Concrete Forms", "Mobile equipment", "Machine", "Electrical", "Tool/Cords", "Building", ] col_w = pw / 3 for row_start in range(0, len(all_types), 3): row_items = all_types[row_start:row_start + 3] for item in row_items: checked = item in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(col_w - 5, 5, " " + item) pdf.ln(5.5) # Other row other_checked = "Other" in issue_types pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5, "4" if other_checked else "", border=1) pdf.set_font("Helvetica", "", 8) other_display = "Other: " + other_text if other_checked and other_text else "Other:" pdf.cell(col_w - 5, 5, " " + other_display) pdf.ln(6) # Best times / Urgency / Desired completion (two-column layout) y_section = pdf.get_y() half = pw / 2 lm = pdf.l_margin # left margin x # -- Left column: best times + desired completion -- pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Best times for service", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("best_times", ""), border="B", ln=1) pdf.ln(2) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Desired completion date", ln=1) pdf.set_font("Helvetica", "", 9) pdf.cell(half, 6, data.get("desired_completion_date", ""), border="B", ln=1) left_end_y = pdf.get_y() # -- Right column: urgency checkboxes -- urgency = data.get("urgency", "") urg_x = lm + half pdf.set_xy(urg_x, y_section) pdf.set_font("Helvetica", "B", 7) pdf.cell(half, 5, "Urgency (check one)", ln=1) pdf.set_x(urg_x) for level in ["Low", "Medium", "High"]: checked = urgency == level pdf.set_font("ZapfDingbats", "", 9) pdf.cell(5, 5.5, "4" if checked else "", border=1) pdf.set_font("Helvetica", "", 8) pdf.cell(half - 5, 5.5, " " + level, ln=1) pdf.set_x(urg_x) right_end_y = pdf.get_y() pdf.set_y(max(left_end_y, right_end_y) + 4) pdf.set_fill_color(*dark_gray) pdf.set_text_color(*white) pdf.set_font("Helvetica", "B", 9) pdf.cell(pw, 7, " For use by Safety department", fill=True, ln=1) pdf.set_text_color(*black) m = safety or {} wo_num = m.get("work_order_number", "") or (f"{data['_wor_number']:05d}" if data.get("_wor_number") else "") label_value_row([ ("Accepted by", m.get("accepted_by", "")), ("Date", m.get("date", "")), ("Assigned work order #", wo_num), ], row_h=10) return pdf.output() @app.route("/wor-submit", methods=["POST"]) @login_required def wor_submit(): """Accept work order form data, generate PDF, and store it.""" if not _wor_type_enabled_for("maintenance", session["user"]): return jsonify(error="This work order type is not currently enabled."), 403 data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 data["_wor_type"] = "maintenance" wor_num = _next_work_order_number() data["_wor_number"] = wor_num WOR_MAINTENANCE_DIR.mkdir(parents=True, exist_ok=True) # Per-WO notification opt-in (unchecked by default). The recipient email # comes from the signed-in user, not the (editable) form field, so opt-in # always notifies the actual submitter. ``email`` mirrors ``requestor_email`` # because the notifications core resolves recipients by the ``email`` key. user = session.get("user") or {} notify_email = user.get("email", "") data["_notify"] = { "enabled": "notify_enabled" in request.form, "requestor_name": user.get("name") or data["requestor_name"], "requestor_email": notify_email, "email": notify_email, "disabled_by_admin": False, } pdf_bytes = _generate_wor_pdf(data) timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", data["requestor_name"])[:40].strip().replace(" ", "_") filename = f"WOR_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_MAINTENANCE_DIR / filename out_path.write_bytes(pdf_bytes) json_path = out_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) # Raise workorder.created — one branded email per opted-in recipient. notifications.notify( notifications.EVENT_WO_CREATED, _wor_event_context(filename, "maintenance", data), ) _send_wor_admin_email_bg( f"New Maintenance Work Order #{wor_num:05d} — {data['requestor_name']}", f"A new Maintenance Work Order has been submitted.\n\n" f"Requestor: {data['requestor_name']}\n" f"Date: {data.get('date_of_request', '')}\n" f"Location: {data.get('location', '')}\n" f"Description: {data.get('description', '')[:200]}\n" f"Urgency: {data.get('urgency', '')}\n" f"Desired Completion: {data.get('desired_completion_date', 'Not set')}\n\n" f"Filename: {filename}\n\n" f"This notification was sent from the ICAST Togen application.", ) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "created", actor=(session.get("user") or {}).get("name"), summary=f"Maintenance WO #{wor_num:05d} created — {data['requestor_name']}", ref=f"WO#{wor_num:05d}", detail={"wor_type": "maintenance", "filename": filename}, ) return jsonify(filename=filename) @app.route("/safety-wor-submit", methods=["POST"]) @login_required def safety_wor_submit(): """Accept Safety Concerns work order form data, generate PDF, and store it.""" if not _wor_type_enabled_for("safety", session["user"]): return jsonify(error="This work order type is not currently enabled."), 403 data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 data["_wor_type"] = "safety" wor_num = _next_work_order_number() data["_wor_number"] = wor_num WOR_SAFETY_DIR.mkdir(parents=True, exist_ok=True) # Per-WO notification opt-in (unchecked by default). Recipient email comes # from the signed-in user, not the editable form field, so opt-in always # notifies the actual submitter. See wor_submit() for the shared shape. user = session.get("user") or {} notify_email = user.get("email", "") data["_notify"] = { "enabled": "notify_enabled" in request.form, "requestor_name": user.get("name") or data["requestor_name"], "requestor_email": notify_email, "email": notify_email, "disabled_by_admin": False, } pdf_bytes = _generate_safety_wor_pdf(data) timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", data["requestor_name"])[:40].strip().replace(" ", "_") filename = f"SWO_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_SAFETY_DIR / filename out_path.write_bytes(pdf_bytes) json_path = out_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) _send_wor_admin_email_bg( f"New Safety Work Order #{wor_num:05d} — {data['requestor_name']}", f"A new Safety Concerns Work Order has been submitted.\n\n" f"Requestor: {data['requestor_name']}\n" f"Date: {data.get('date_of_request', '')}\n" f"Location: {data.get('location', '')}\n" f"Description: {data.get('description', '')[:200]}\n" f"Urgency: {data.get('urgency', '')}\n" f"Desired Completion: {data.get('desired_completion_date', 'Not set')}\n\n" f"Filename: {filename}\n\n" f"This notification was sent from the ICAST Togen application.", ) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "created", actor=(session.get("user") or {}).get("name"), summary=f"Safety WO #{wor_num:05d} created — {data['requestor_name']}", ref=f"WO#{wor_num:05d}", detail={"wor_type": "safety", "filename": filename}, ) return jsonify(filename=filename) @app.route("/mm-wor-submit", methods=["POST"]) @login_required def mm_wor_submit(): """Accept Mobile Machine work order form data, generate PDF, and store it.""" if not _wor_type_enabled_for("mobilemachine", session["user"]): return jsonify(error="This work order type is not currently enabled."), 403 part_numbers = request.form.getlist("part_number[]") part_names = request.form.getlist("part_name[]") part_qtys = request.form.getlist("part_qty[]") part_amounts = request.form.getlist("part_amount[]") parts = [] for i in range(len(part_numbers)): pn = part_numbers[i].strip() if i < len(part_numbers) else "" nm = part_names[i].strip() if i < len(part_names) else "" qt = part_qtys[i].strip() if i < len(part_qtys) else "" am = part_amounts[i].strip() if i < len(part_amounts) else "" if pn or nm or qt or am: parts.append({"number": pn, "name": nm, "qty": qt, "amount": am}) parts = _rpm_register_submitted_parts(parts, "mobilemachine", session.get("user")) data = { "machine": request.form.get("machine", "").strip(), "asset_number": request.form.get("asset_number", "").strip(), "order_date_time": request.form.get("order_date_time", "").strip(), "date_promised": request.form.get("date_promised", "").strip(), "date_delivered": request.form.get("date_delivered", "").strip(), "order_received_by": request.form.get("order_received_by", "").strip(), "make_model": request.form.get("make_model", "").strip(), "vin": request.form.get("vin", "").strip(), "odometer": request.form.get("odometer", "").strip(), "license_state": request.form.get("license_state", "").strip(), "motor_number": request.form.get("motor_number", "").strip(), "service_type": request.form.getlist("service_type"), "labor_description": request.form.get("labor_description", "").strip(), "labor_time": request.form.get("labor_time", "").strip(), "labor_total": request.form.get("labor_total", "").strip(), "parts": parts, "awaiting_approval": "awaiting_approval" in request.form, "parts_ordered": "parts_ordered" in request.form, "parts_total": request.form.get("parts_total", "").strip(), "compiled_by": request.form.get("compiled_by", "").strip(), "compiled_date": request.form.get("compiled_date", "").strip(), "authorized_by": request.form.get("authorized_by", "").strip(), "authorization_date": request.form.get("authorization_date", "").strip(), "_wor_type": "mobilemachine", } if not data["machine"]: return jsonify(error="Machine name is required"), 400 # Optional manual attachment (DVI-462), indexed under the machine name. manual_info, manual_err = _pm_process_manual(data["machine"], session.get("user") or {}) if manual_err: return jsonify(error=manual_err), 400 if manual_info: data["manual"] = manual_info wor_num = _next_work_order_number() data["_wor_number"] = wor_num WOR_MOBILEMACHINE_DIR.mkdir(parents=True, exist_ok=True) # Per-WO notification opt-in (unchecked by default). Recipient email comes # from the signed-in user, not a form field, so opt-in always notifies the # actual submitter. See wor_submit() for the shared shape. user = session.get("user") or {} notify_email = user.get("email", "") data["_notify"] = { "enabled": "notify_enabled" in request.form, "requestor_name": user.get("name") or data["machine"], "requestor_email": notify_email, "email": notify_email, "disabled_by_admin": False, } pdf_bytes = _generate_mm_wor_pdf(data) timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_machine = re.sub(r"[^\w\s\-]", "", data["machine"])[:40].strip().replace(" ", "_") filename = f"MMWO_{wor_num:05d}_{timestamp}_{safe_machine}.pdf" out_path = WOR_MOBILEMACHINE_DIR / filename out_path.write_bytes(pdf_bytes) json_path = out_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) _send_wor_admin_email_bg( f"New Mobile Machine Work Order #{wor_num:05d} — {data['machine']}", f"A new Mobile Machine Work Order has been submitted.\n\n" f"Machine: {data['machine']}\n" f"Asset #: {data.get('asset_number', '')}\n" f"Order Date: {data.get('order_date_time', '')}\n" f"Date Promised: {data.get('date_promised', 'Not set')}\n" f"Make/Model: {data.get('make_model', '')}\n\n" f"Filename: {filename}\n\n" f"This notification was sent from the ICAST Togen application.", ) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "created", actor=(session.get("user") or {}).get("name"), summary=f"Mobile Machine WO #{wor_num:05d} created — {data['machine']}", ref=f"WO#{wor_num:05d}", detail={"wor_type": "mobilemachine", "filename": filename}, ) return jsonify(filename=filename) @app.route("/preventive-wor-submit", methods=["POST"]) @login_required def preventive_wor_submit(): """Accept Preventative Maintenance work order form data, generate PDF, and store it.""" if not _wor_type_enabled_for("preventive", session["user"]): return jsonify(error="This work order type is not currently enabled."), 403 part_numbers = request.form.getlist("part_number[]") part_names = request.form.getlist("part_name[]") part_qtys = request.form.getlist("part_qty[]") part_amounts = request.form.getlist("part_amount[]") parts = [] for i in range(len(part_numbers)): pn = part_numbers[i].strip() if i < len(part_numbers) else "" nm = part_names[i].strip() if i < len(part_names) else "" qt = part_qtys[i].strip() if i < len(part_qtys) else "" am = part_amounts[i].strip() if i < len(part_amounts) else "" if pn or nm or qt or am: parts.append({"number": pn, "name": nm, "qty": qt, "amount": am}) parts = _rpm_register_submitted_parts(parts, "preventive", session.get("user")) data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), "parts": parts, "awaiting_approval": "awaiting_approval" in request.form, "parts_ordered": "parts_ordered" in request.form, "parts_total": request.form.get("parts_total", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 manual_info, manual_err = _pm_process_manual(data["equipment"], session.get("user") or {}) if manual_err: return jsonify(error=manual_err), 400 if manual_info: data["manual"] = manual_info data["_wor_type"] = "preventive" wor_num = _next_work_order_number() data["_wor_number"] = wor_num WOR_PREVENTIVE_DIR.mkdir(parents=True, exist_ok=True) user = session.get("user") or {} notify_email = user.get("email", "") data["_notify"] = { "enabled": "notify_enabled" in request.form, "requestor_name": user.get("name") or data["requestor_name"], "requestor_email": notify_email, "email": notify_email, "disabled_by_admin": False, } pdf_bytes = _generate_preventive_wor_pdf(data) timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", data["requestor_name"])[:40].strip().replace(" ", "_") filename = f"PMWO_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_PREVENTIVE_DIR / filename out_path.write_bytes(pdf_bytes) json_path = out_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) notifications.notify( notifications.EVENT_WO_CREATED, _wor_event_context(filename, "preventive", data), ) _send_wor_admin_email_bg( f"New Preventative Maintenance Work Order #{wor_num:05d} — {data['requestor_name']}", f"A new Preventative Maintenance Work Order has been submitted.\n\n" f"Requestor: {data['requestor_name']}\n" f"Date: {data.get('date_of_request', '')}\n" f"Location: {data.get('location', '')}\n" f"Description: {data.get('description', '')[:200]}\n" f"Urgency: {data.get('urgency', '')}\n" f"Desired Completion: {data.get('desired_completion_date', 'Not set')}\n\n" f"Filename: {filename}\n\n" f"This notification was sent from the ICAST Togen application.", ) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "created", actor=(session.get("user") or {}).get("name"), summary=f"Preventive Maint. WO #{wor_num:05d} created — {data['requestor_name']}", ref=f"WO#{wor_num:05d}", detail={"wor_type": "preventive", "filename": filename}, ) return jsonify(filename=filename, manual=data.get("manual")) @app.route("/wor-history") @login_required def wor_history(): """List new and in-progress Work Order Request PDFs (excludes completed).""" wor_type = request.args.get("type", "maintenance") dirs = _wor_dest_dirs(wor_type) files = [] for directory, status in [(dirs["new"], "new"), (dirs["in_progress"], "in_progress")]: if not directory.is_dir(): continue for f in directory.glob("*.pdf"): mtime = _datetime_type.utcfromtimestamp(f.stat().st_mtime) sidecar = _resolve_wor_sidecar(f.name, wor_type) data = None if sidecar: try: data = json.loads(sidecar.read_text()) except (ValueError, OSError): data = None files.append({ "name": f.name, "title": _wor_display_title(f.name, wor_type, data), "date": mtime.strftime("%Y-%m-%dT%H:%M:%SZ"), "status": status, }) files.sort(key=lambda x: x["date"], reverse=True) return jsonify(files=files) @app.route("/wor-preview") @login_required def wor_preview(): """Serve a Work Order PDF inline for preview.""" filename = request.args.get("file", "") file_path = _resolve_wor_file(filename) if not file_path: return "File not found", 404 return send_file(str(file_path), mimetype="application/pdf") @app.route("/wor-download") @login_required def wor_download(): """Download a Work Order Request PDF.""" filename = request.args.get("file", "") file_path = _resolve_wor_file(filename) if not file_path: return "File not found", 404 return send_file(str(file_path), as_attachment=True, download_name=filename) @app.route("/wor-viewer-url") @login_required def wor_viewer_url(): """Return the Document Viewer URL for a work order PDF.""" filename = request.args.get("file", "") file_path = _resolve_wor_file(filename) if not file_path: return jsonify({"error": "not found"}), 404 try: rel = file_path.relative_to(WOR_DIR.resolve()) except ValueError: return jsonify({"error": "not found"}), 404 prefix = _load_shared_url_prefix() url = f"{prefix}/wor/{rel}" return jsonify({"url": url}) @app.route("/pm-manual-lookup") @login_required def pm_manual_lookup(): """Return any manual previously associated with an Equipment/Asset/System name.""" entry = _manuals_index_lookup(request.args.get("equipment", "")) if not entry: return jsonify({"found": False}) return jsonify({ "found": True, "filename": entry["filename"], "original_name": entry.get("original_name", entry["filename"]), "url": entry.get("url", _manual_viewer_url(entry["filename"])), "equipment": entry.get("equipment", ""), }) @app.route("/pm-eas-options") @login_required def pm_eas_options(): """List known Equipment/Asset/System names for the PM work order type-ahead (DVI-590).""" return jsonify({"options": _collect_pm_eas_options()}) @app.route("/pm-manual-url") @login_required def pm_manual_url(): """Return the Document Viewer URL for a stored manual file.""" filename = (request.args.get("file", "") or "").strip() if not filename or "/" in filename or "\\" in filename or ".." in filename: return jsonify({"error": "not found"}), 404 if not (WOR_MANUALS_DIR / filename).is_file(): return jsonify({"error": "not found"}), 404 return jsonify({"url": _manual_viewer_url(filename)}) @app.route("/wor-manual-info") @login_required def wor_manual_info(): """Return the manual attached to a given work order (from its JSON sidecar).""" sidecar = _resolve_wor_sidecar(request.args.get("file", "")) if not sidecar: return jsonify({"found": False}) try: data = json.loads(sidecar.read_text()) except (json.JSONDecodeError, OSError): return jsonify({"found": False}) manual = data.get("manual") or {} stored = manual.get("filename") if not stored or not (WOR_MANUALS_DIR / stored).is_file(): return jsonify({"found": False}) return jsonify({ "found": True, "filename": stored, "original_name": manual.get("original_name", stored), "url": _manual_viewer_url(stored), }) @app.route("/equipment-manuals/list") @login_required def equipment_manuals_list(): """Return all uploaded/attached equipment manuals for the Equipment Manuals tool.""" return jsonify({"manuals": _list_all_manuals()}) def _em_search_manuals(q: str, only_filename: str | None = None) -> list: """Full-text OCR search across stored manuals (DVI-491). Returns each matching manual with a context snippet, the 1-based page number of the first hit, total hit count, and a link to open it. When ``only_filename`` is given, the search is restricted to that single manual (DVI-626: WO-scoped "start with the attached manual" search).""" ql = (q or "").strip().lower() if len(ql) < 2: return [] # Map stored filename -> association entry for equipment name / open URL. assoc = {} for entry in _load_manuals_index().values(): fn = entry.get("filename") if fn: assoc[fn] = entry results = [] for fn, entry in _load_manuals_text_index().items(): if only_filename and fn != only_filename: continue if not (WOR_MANUALS_DIR / fn).is_file(): continue text = entry.get("text", "") or "" tl = text.lower() first = tl.find(ql) if first < 0: continue count = tl.count(ql) # First-hit page number via the per-page offset table. page = 1 for p in entry.get("pages", []): if p.get("start", 0) <= first < p.get("end", 0): page = p.get("number", 1) break start = max(0, first - 60) end = min(len(text), first + len(ql) + 60) snippet = text[start:end].replace("\f", " ").strip() if start > 0: snippet = "…" + snippet if end < len(text): snippet = snippet + "…" meta = assoc.get(fn, {}) results.append({ "filename": fn, "original_name": entry.get("original_name") or meta.get("original_name", fn), "equipment": meta.get("equipment", ""), "url": _manual_viewer_url(fn), "page": page, "match_count": count, "snippet": snippet, }) results.sort(key=lambda r: r["match_count"], reverse=True) return results @app.route("/equipment-manuals/search") @login_required def equipment_manuals_search(): """Cross-manual full-text search over OCR'd manuals (DVI-491, D2). Within-document search is served separately by the searchable-PDF text layer (Ctrl-F).""" q = (request.args.get("q") or "").strip() if len(q) < 2: return jsonify(results=[], query=q) return jsonify(results=_em_search_manuals(q), query=q) @app.route("/equipment-manuals/file/") @login_required def equipment_manuals_file(stored_filename): """Serve a manual's raw bytes same-origin (DVI-464). The public ``url`` points at the external Document Viewer (docs.icastinc.com), which is cross-origin and therefore unusable as a PDF.js / source for the Grid-view thumbnails. This route streams the same file from the local manuals store so the browser can render a first-page preview without CORS issues. """ if "/" in stored_filename or "\\" in stored_filename or ".." in stored_filename: abort(404) path = WOR_MANUALS_DIR / stored_filename if not path.is_file(): abort(404) return send_file(str(path)) @app.route("/equipment-manuals/thumb/") @login_required def equipment_manuals_thumb(stored_filename): """Serve a small cached Grid-view thumbnail for a manual (DVI-585). Returns a pre-rendered JPEG (page 1 for PDFs, downscaled for images) so the grid loads quickly instead of downloading + rendering each full manual. A 404 here is expected for unsupported types or when the renderer is missing; the client falls back to its own pdf.js/image rendering. """ if "/" in stored_filename or "\\" in stored_filename or ".." in stored_filename: abort(404) thumb = _ensure_manual_thumb(stored_filename) if not thumb: abort(404) resp = send_file(str(thumb), mimetype="image/jpeg") resp.headers["Cache-Control"] = "private, max-age=86400" return resp @app.route("/equipment-manuals/upload", methods=["POST"]) @login_required def equipment_manuals_upload(): """Upload a manual and associate it with an Equipment/Asset/System name. The equipment name becomes the manual's attribution key, so a later Preventative Maintenance work order naming the same equipment auto-suggests this manual (see /pm-manual-lookup). """ equipment = (request.form.get("equipment", "") or "").strip() if not equipment: return jsonify(error="Equipment / Asset / System name is required"), 400 f = request.files.get("manual") if not f or not f.filename: return jsonify(error="A manual file is required"), 400 ext = os.path.splitext(f.filename)[1].lower() if ext not in ALLOWED_MANUAL_EXTS: allowed = ", ".join(sorted(ALLOWED_MANUAL_EXTS)) return jsonify(error=f"Unsupported manual file type. Allowed: {allowed}"), 400 WOR_MANUALS_DIR.mkdir(parents=True, exist_ok=True) base = secure_filename(f.filename) or ("manual" + ext) stored = f"{_datetime_type.now().strftime('%Y%m%d_%H%M%S')}_{base}" f.save(str(WOR_MANUALS_DIR / stored)) _manuals_index_set(equipment, stored, f.filename, session.get("user") or {}) # Background OCR (D3): make the PDF searchable in place + index its text so # it surfaces in cross-manual search. No-op for non-PDFs / missing engine. _enqueue_manual_ocr(stored, f.filename) return jsonify({ "ok": True, "filename": stored, "original_name": f.filename, "equipment": equipment, "url": _manual_viewer_url(stored), }) @app.route("/equipment-manuals/rename", methods=["POST"]) @login_required def equipment_manuals_rename(): """Edit the Equipment/Asset/System name a manual is associated with (DVI-462). The index is keyed by normalized equipment name, so renaming re-keys the entry (moving it off the old key) while keeping the same stored file. """ body = request.get_json(silent=True) or {} filename = (body.get("filename") or "").strip() new_eq = (body.get("equipment") or "").strip() if not filename or "/" in filename or "\\" in filename or ".." in filename: return jsonify(error="Invalid manual reference"), 400 if not new_eq: return jsonify(error="Equipment / Asset / System name is required"), 400 if not (WOR_MANUALS_DIR / filename).is_file(): return jsonify(error="Manual file not found"), 404 new_key = _normalize_equipment(new_eq) if not new_key: return jsonify(error="Equipment / Asset / System name is required"), 400 index = _load_manuals_index() old_key, entry = None, None for k, v in index.items(): if v.get("filename") == filename: old_key, entry = k, v break base = dict(entry) if entry else {"filename": filename, "original_name": filename} user = session.get("user") or {} base.update({ "filename": filename, "equipment": new_eq, "url": _manual_viewer_url(filename), "updated_at": _datetime_type.now().strftime("%Y-%m-%dT%H:%M:%S"), "updated_by": user.get("name", ""), }) if old_key is not None and old_key != new_key: index.pop(old_key, None) index[new_key] = base _save_manuals_index(index) return jsonify(ok=True, filename=filename, equipment=new_eq, url=base["url"]) @app.route("/wor-attach-manual", methods=["POST"]) @login_required def wor_attach_manual(): """Attach a manual to an existing work order's JSON sidecar (DVI-462). Accepts a multipart form: ``file`` is the target WO's PDF filename. The manual itself is resolved via the same logic as PM submission — a freshly uploaded ``manual`` file (with an ``equipment`` name to index it under) takes precedence, otherwise an existing manual referenced by ``manual_existing``. The 'Open Manual' controls in WOA/WOR read this sidecar ``manual`` key. """ filename = (request.form.get("file") or "").strip() sidecar = _resolve_wor_sidecar(filename) if not sidecar: return jsonify(error="Work order not found"), 404 equipment = (request.form.get("equipment") or "").strip() manual_info, err = _pm_process_manual(equipment, session.get("user") or {}) if err: return jsonify(error=err), 400 if not manual_info: return jsonify(error="No manual provided"), 400 try: data = json.loads(sidecar.read_text()) except (json.JSONDecodeError, OSError): return jsonify(error="Work order data could not be read"), 500 data["manual"] = manual_info sidecar.write_text(json.dumps(data, indent=2)) return jsonify(ok=True, filename=manual_info["filename"], original_name=manual_info.get("original_name"), url=manual_info.get("url")) @app.route("/wor-email", methods=["POST"]) @login_required def wor_email(): """Email a Work Order Request PDF via Microsoft Graph API.""" body = request.get_json(silent=True) or {} filename = body.get("filename", "") to_email = body.get("email", "").strip() if not filename: return jsonify(error="Invalid filename"), 400 if not to_email: return jsonify(error="Email address is required"), 400 file_path = _resolve_wor_file(filename) if not file_path: return jsonify(error="File not found"), 404 if not AZURE_MAIL_FROM: return jsonify(error="Email sender is not configured. Set AZURE_MAIL_FROM."), 500 # Acquire an application token via client credentials flow app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire mail token: {err}"), 500 pdf_bytes = file_path.read_bytes() wor_type = _detect_wor_type(file_path) if wor_type == "mobilemachine": subject_prefix = "ICAST Mobile Machine Work Order" elif wor_type == "safety": subject_prefix = "ICAST Safety Work Order" elif wor_type == "preventive": subject_prefix = "ICAST Preventative Maintenance Work Order" else: subject_prefix = "ICAST Maintenance Work Order" graph_body = { "message": { "subject": f"{subject_prefix} — {filename}", "body": { "contentType": "Text", "content": ( "Please find the attached Work Order PDF.\n\n" "This email was sent from the ICAST Togen application." ), }, "toRecipients": [ {"emailAddress": {"address": to_email}}, ], "attachments": [ { "@odata.type": "#microsoft.graph.fileAttachment", "name": filename, "contentType": "application/pdf", "contentBytes": base64.b64encode(pdf_bytes).decode(), }, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code not in (200, 202): detail = resp.json().get("error", {}).get("message", resp.text[:200]) return jsonify(error=f"Graph API error: {detail}"), 502 return jsonify(ok=True) # --------------------------------------------------------------------------- # Routes — Work Order Admin (DVI-233) # --------------------------------------------------------------------------- @app.route("/wor-admin-list") @wor_admin_required def wor_admin_list(): """List all WORs across new / in_progress / completed with status and type.""" registry = _load_recurring_registry() files = [] for wor_type in ("maintenance", "mobilemachine", "safety", "preventive"): dirs = _wor_dest_dirs(wor_type) for status_key, directory in dirs.items(): if not directory.is_dir(): continue for f in directory.glob("*.pdf"): mtime = _datetime_type.utcfromtimestamp(f.stat().st_mtime) entry = { "name": f.name, "date": mtime.strftime("%Y-%m-%dT%H:%M:%SZ"), "status": status_key, "wor_type": wor_type, "mtime": f.stat().st_mtime, } rec = registry.get(f.name) if rec: entry["recurring"] = { "frequency_days": rec.get("frequency_days"), "next_run": rec.get("next_run", ""), "end_date": rec.get("end_date", ""), "enabled": rec.get("enabled", True), } sidecar = _resolve_wor_sidecar(f.name, wor_type) sidecar_data = None if sidecar: try: sidecar_data = json.loads(sidecar.read_text()) except (ValueError, OSError): sidecar_data = None notify = (sidecar_data or {}).get("_notify") or {} if notify.get("enabled"): entry["notify"] = { "enabled": True, "requestor_name": notify.get("requestor_name", ""), } entry["submitter_email"] = (sidecar_data or {}).get("email", "") entry["submitter_name"] = (sidecar_data or {}).get("requestor_name", "") # Parts count drives the WO Admin Parts icon (DVI-604); only MM/PM # WOs carry repair parts. Read-only count — no sidecar write here. if wor_type in RPM_PART_WOR_TYPES: parts_norm, _ = _rpm_normalize_parts((sidecar_data or {}).get("parts")) entry["parts_count"] = len(parts_norm) entry["title"] = _wor_display_title(f.name, wor_type, sidecar_data) files.append(entry) files.sort(key=lambda x: x["mtime"], reverse=True) for f in files: del f["mtime"] return jsonify(files=files) @app.route("/wor-admin-activity") @wor_admin_required def wor_admin_activity(): """Activity metrics for the Work Order Admin dashboard (DVI-380). Aggregates from existing files only (no new storage): a 30-day zero-filled submission histogram, status counts across all three WOR types, and recurring schedules due today or tomorrow (from the recurring registry). """ from datetime import timedelta today = _datetime_type.now().date() # 30-day zero-filled histogram anchored on the server's local "today". window_start = today - timedelta(days=29) day_counts = { (window_start + timedelta(days=i)).strftime("%Y-%m-%d"): 0 for i in range(30) } status_counts = {"new": 0, "in_progress": 0, "completed": 0, "cancelled": 0} for wor_type in ("maintenance", "mobilemachine", "safety", "preventive"): for status_key, directory in _wor_dest_dirs(wor_type).items(): if not directory.is_dir(): continue for f in directory.glob("*.pdf"): status_counts[status_key] = status_counts.get(status_key, 0) + 1 # Prefer the recorded submission date from the sidecar # (`date` on recurring-generated WOs, `date_of_request` on # form submissions); fall back to file mtime when absent. day = "" sidecar = f.with_suffix(".json") if sidecar.is_file(): try: sd = json.loads(sidecar.read_text()) or {} day = (sd.get("date") or sd.get("date_of_request") or "").strip() except (ValueError, OSError): day = "" if not day: day = _datetime_type.fromtimestamp( f.stat().st_mtime).strftime("%Y-%m-%d") if day in day_counts: day_counts[day] += 1 submissions_by_day = [ {"date": d, "count": c} for d, c in sorted(day_counts.items()) ] # Recurring schedules due today (days_until 0) or tomorrow (1). Overdue # stragglers (negative days_until) are surfaced too — they're past due. recurring_due_soon = [] for filename, rec in _load_recurring_registry().items(): if not rec.get("enabled", True): continue next_run = rec.get("next_run") or "" try: nr = _datetime_type.strptime(next_run, "%Y-%m-%d").date() except (ValueError, TypeError): continue days_until = (nr - today).days if days_until <= 1: recurring_due_soon.append({ "name": filename, "next_run": next_run, "days_until": days_until, "wor_type": rec.get("wor_type", "maintenance"), }) recurring_due_soon.sort(key=lambda r: r["days_until"]) return jsonify( submissions_by_day=submissions_by_day, status_counts=status_counts, recurring_due_soon=recurring_due_soon, ) @app.route("/wor-admin-toggle-notify", methods=["POST"]) @wor_admin_required def wor_admin_toggle_notify(): """Admin disables per-WO notifications: sets _notify.enabled false and flags it as admin-disabled.""" body = request.get_json(silent=True) or {} filename = body.get("filename", "") if not filename or ".." in filename or "/" in filename: return jsonify(error="Invalid filename"), 400 sidecar = _resolve_wor_sidecar(filename) if not sidecar: return jsonify(error="File not found"), 404 try: data = json.loads(sidecar.read_text()) or {} except (ValueError, OSError): return jsonify(error="Could not read work order data"), 500 notify = data.get("_notify") or {} notify["enabled"] = False notify["disabled_by_admin"] = True data["_notify"] = notify sidecar.write_text(json.dumps(data, indent=2)) return jsonify(ok=True) # ----- Repair Parts Manager (RPM) backend — DVI-596 --------------------------- # Option A data model: parts stay on each Work Order sidecar JSON (under "parts"). # RPM extends each part with a stable "id" and "approved"/"ordered" booleans, and # keeps a separate Part Number -> Part Name catalog (RPM_KNOWN_PARTS_FILE) for # autofill. Aggregation views scan the Mobile Machine + Preventative Maintenance # WO directories, mirroring how wor_admin_list reads sidecars. def _rpm_load_known_parts() -> dict: """Return the {part_number: part_name} autofill catalog.""" if RPM_KNOWN_PARTS_FILE.is_file(): try: d = json.loads(RPM_KNOWN_PARTS_FILE.read_text()) if isinstance(d, dict): return d except (ValueError, OSError): pass return {} def _rpm_save_known_parts(catalog: dict) -> None: RPM_KNOWN_PARTS_FILE.write_text(json.dumps(catalog, indent=2)) def _rpm_upsert_known_part(number: str, name: str) -> None: """Remember a Part Number -> Part Name mapping (latest edit wins).""" number = (number or "").strip() name = (name or "").strip() if not number or not name: return catalog = _rpm_load_known_parts() if catalog.get(number) != name: catalog[number] = name _rpm_save_known_parts(catalog) # RPM Settings (DVI-607). Policy toggles that change part-processing behavior. _RPM_SETTINGS_DEFAULTS = { "assume_stocked_on_complete": False, # DVI-609: auto-mark a part Stocked once its carrier tracking reports # delivered. The carrier polling that sets tracking_status is deferred, but # the policy applies the moment a part's status is "delivered". "mark_stocked_on_delivered": False, } def _rpm_load_settings() -> dict: settings = dict(_RPM_SETTINGS_DEFAULTS) if RPM_SETTINGS_FILE.is_file(): try: d = json.loads(RPM_SETTINGS_FILE.read_text()) if isinstance(d, dict): for k in _RPM_SETTINGS_DEFAULTS: if k in d: settings[k] = bool(d[k]) except (ValueError, OSError): pass return settings def _rpm_save_settings(settings: dict) -> None: RPM_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) # RPM Approval permission + Auto Approval policy (DVI-614). # approve_mode — who may approve parts (Settings > Policy > Approval permission): # "all_admins" (any Work Order Admin), "primary" (only emails on the Primary # list that enforces notifications, per WO type), or "specific" (an explicit # email allowlist). # approve_users — allowlist used when approve_mode == "specific". # auto_approval — master ``enabled`` plus a per-WO-type block. Each type block # has ``mode`` ("approvers" = the approve_mode designation, "all_admins", # or "specific") and a ``users`` allowlist for the "specific" mode. When a # part is added/submitted by a user matching the type's rule, it is auto # approved. _RPM_APPROVE_MODES = ("all_admins", "primary", "specific") _RPM_AUTO_MODES = ("approvers", "all_admins", "specific") _RPM_APPROVAL_DEFAULTS = { "approve_mode": "all_admins", "approve_users": [], "auto_approval": { "enabled": False, "mobilemachine": {"mode": "approvers", "users": []}, "preventive": {"mode": "approvers", "users": []}, }, } def _rpm_norm_emails(values) -> list: """Lower-cased, de-duplicated, order-stable email list from raw input.""" out, seen = [], set() for v in (values or []): e = str(v or "").strip().lower() if e and e not in seen: seen.add(e) out.append(e) return out def _rpm_load_approval() -> dict: """Return the RPM approval/auto-approval policy, defaults filled in.""" pol = { "approve_mode": "all_admins", "approve_users": [], "auto_approval": { "enabled": False, "mobilemachine": {"mode": "approvers", "users": []}, "preventive": {"mode": "approvers", "users": []}, }, # DVI-614 follow-up: per-WO-type email allowlists. A user on a type's # list receives ALL RPM notifications for that type as a matter of # policy, regardless of their per-user notifications grid. "policy_notify": {wt: [] for wt in RPM_PART_WOR_TYPES}, } if RPM_APPROVAL_FILE.is_file(): try: d = json.loads(RPM_APPROVAL_FILE.read_text()) except (ValueError, OSError): d = {} if isinstance(d, dict): if d.get("approve_mode") in _RPM_APPROVE_MODES: pol["approve_mode"] = d["approve_mode"] pol["approve_users"] = _rpm_norm_emails(d.get("approve_users")) auto = d.get("auto_approval") if isinstance(auto, dict): pol["auto_approval"]["enabled"] = bool(auto.get("enabled")) for wt in RPM_PART_WOR_TYPES: blk = auto.get(wt) if isinstance(blk, dict): mode = blk.get("mode") if mode in _RPM_AUTO_MODES: pol["auto_approval"][wt]["mode"] = mode pol["auto_approval"][wt]["users"] = _rpm_norm_emails(blk.get("users")) pn = d.get("policy_notify") if isinstance(pn, dict): for wt in RPM_PART_WOR_TYPES: pol["policy_notify"][wt] = _rpm_norm_emails(pn.get(wt)) return pol def _rpm_save_approval(pol: dict) -> None: RPM_APPROVAL_FILE.write_text(json.dumps(pol, indent=2)) def _rpm_user_can_approve(user: dict, wor_type: str) -> bool: """Whether ``user`` (session dict) may approve parts for ``wor_type`` under the current Approval permission policy. Togen Admins always may.""" if not isinstance(user, dict): return False if user.get("togen_admin"): return True pol = _rpm_load_approval() mode = pol.get("approve_mode", "all_admins") email = (user.get("email") or "").strip().lower() if mode == "primary": primary = _load_wor_primary() return email in [e.lower() for e in primary.get(wor_type, [])] if mode == "specific": return email in pol.get("approve_users", []) # "all_admins" (default) return bool(user.get("wor_admin")) def _rpm_auto_approve_for(user: dict, wor_type: str) -> bool: """Whether a part added/submitted by ``user`` for ``wor_type`` should be auto approved under the current Auto Approval policy.""" if not isinstance(user, dict): return False pol = _rpm_load_approval() auto = pol.get("auto_approval") or {} if not auto.get("enabled"): return False cfg = auto.get(wor_type) or {} mode = cfg.get("mode", "approvers") email = (user.get("email") or "").strip().lower() if mode == "all_admins": return bool(user.get("wor_admin") or user.get("togen_admin")) if mode == "specific": return email in _rpm_norm_emails(cfg.get("users")) # "approvers" — anyone who satisfies the approve permission designation. return _rpm_user_can_approve(user, wor_type) def _rpm_notify_part(event, wor_type, filename, data, part) -> None: """Fire a Repair Parts Manager part-lifecycle email (DVI-614) to users who opted into (wor_type x event) in their RPM notifications grid. No-ops when no one is subscribed; runs on a background thread.""" try: notifications.notify_rpm(event, { "wor_type": wor_type, "wo_label": _wor_number_label(filename, data), "wo_title": _wor_display_title(filename, wor_type, data), "part_name": (part or {}).get("name") or "", "part_number": (part or {}).get("number") or "", }) except Exception: # notifications must never break the request path pass def _rpm_effective_stocked(part, status_key, assume_on_complete, mark_on_delivered=False) -> bool: """A part counts as stocked when explicitly flagged, or (per the "assume stocked on complete" policy) when its WO is marked completed, or (per the "mark stocked on delivered" policy, DVI-609) when its carrier tracking status is delivered.""" if bool(part.get("stocked")): return True if assume_on_complete and status_key == "completed": return True if mark_on_delivered and str(part.get("tracking_status") or "").lower() == "delivered": return True return False def _rpm_normalize_parts(parts): """Return (normalized_parts, changed) — every part gets a stable id plus approved/ordered/stocked booleans, preserving number/name/qty/amount.""" normalized, changed, seen = [], False, set() for p in (parts or []): if not isinstance(p, dict): changed = True continue pid = str(p.get("id") or "").strip() if not pid or pid in seen: pid = f"rpm_{uuid.uuid4().hex[:12]}" changed = True seen.add(pid) if "id" not in p or "approved" not in p or "ordered" not in p or "stocked" not in p: changed = True if "tracking_number" not in p: changed = True normalized.append({ "id": pid, "number": str(p.get("number") or "").strip(), "name": str(p.get("name") or "").strip(), "qty": str(p.get("qty") or "").strip(), "amount": str(p.get("amount") or "").strip(), "approved": bool(p.get("approved", False)), "ordered": bool(p.get("ordered", False)), "stocked": bool(p.get("stocked", False)), # DVI-609 tracking fields. tracking_status/delivered_at are populated # by the (deferred) carrier integration; "delivered" drives the # mark-stocked-on-delivered policy in _rpm_effective_stocked. "tracking_number": str(p.get("tracking_number") or "").strip(), "carrier": str(p.get("carrier") or "").strip(), "tracking_status": str(p.get("tracking_status") or "").strip(), "delivered_at": str(p.get("delivered_at") or "").strip(), }) return normalized, changed def _rpm_register_submitted_parts(parts, wor_type=None, user=None): """Phase 4 WO-Request coupling: normalize parts submitted on the Mobile Machine / Preventative Maintenance request form into the RPM shape (stable id + approved/ordered defaults) and remember each Part Number -> Part Name in the known-parts catalog so they surface in RPM and the WO Admin Parts dialog. Returns the normalized list to store on the WO sidecar. DVI-614: when ``wor_type`` + ``user`` are supplied and the WO creator matches the Auto Approval policy for that type, every submitted part is approved up front. The created WO file does not exist yet here, so part-lifecycle emails are not fired from this helper — the submit handler owns notification.""" normalized, _ = _rpm_normalize_parts(parts) auto = bool(wor_type and user and _rpm_auto_approve_for(user, wor_type)) for p in normalized: if auto: p["approved"] = True _rpm_upsert_known_part(p["number"], p["name"]) return normalized def _rpm_load_wo(filename: str, wor_type: str | None = None): """Resolve a WO sidecar and load its data; returns (sidecar_path, data).""" sidecar = _resolve_wor_sidecar(filename, wor_type) if not sidecar: return None, None try: data = json.loads(sidecar.read_text()) or {} except (ValueError, OSError): return None, None return sidecar, data def _rpm_wo_parts(sidecar, data, persist: bool = True): """Return normalized parts for a WO, persisting backfilled ids/flags once.""" parts, changed = _rpm_normalize_parts(data.get("parts")) if changed and persist and sidecar is not None: data["parts"] = parts try: sidecar.write_text(json.dumps(data, indent=2)) except OSError: pass return parts def _rpm_save_wo_parts(sidecar, data, parts) -> None: data["parts"] = parts sidecar.write_text(json.dumps(data, indent=2)) def _rpm_part_matches(part, approved_f: str, ordered_f: str) -> bool: if approved_f == "yes" and not part.get("approved"): return False if approved_f == "no" and part.get("approved"): return False if ordered_f == "yes" and not part.get("ordered"): return False if ordered_f == "no" and part.get("ordered"): return False return True @app.route("/rpm/parts") @rpm_required def rpm_parts(): """Work Orders (Mobile Machine + Preventative Maintenance) that have parts, grouped WO -> parts. Filters: type (all|mobilemachine|preventive), approved and ordered (all|yes|no), completed (yes|no — default no hides completed WOs). Cancelled WOs are never listed. Only WOs with >=1 matching part are returned.""" type_filter = (request.args.get("type") or "all").strip() approved_f = (request.args.get("approved") or "all").strip() ordered_f = (request.args.get("ordered") or "all").strip() include_completed = (request.args.get("completed") or "no").strip() == "yes" settings = _rpm_load_settings() assume = settings["assume_stocked_on_complete"] mark_delivered = settings["mark_stocked_on_delivered"] types = (type_filter,) if type_filter in RPM_PART_WOR_TYPES else RPM_PART_WOR_TYPES user = session.get("user") or {} can_approve_by_type = {wt: _rpm_user_can_approve(user, wt) for wt in RPM_PART_WOR_TYPES} groups = [] for wor_type in types: for status_key, directory in _wor_dest_dirs(wor_type).items(): if status_key == "cancelled": continue if status_key == "completed" and not include_completed: continue if not directory.is_dir(): continue for f in directory.glob("*.pdf"): sidecar = f.with_suffix(".json") if not sidecar.is_file(): continue try: data = json.loads(sidecar.read_text()) or {} except (ValueError, OSError): continue parts = _rpm_wo_parts(sidecar, data) if not parts: continue shown = [] for p in parts: if not _rpm_part_matches(p, approved_f, ordered_f): continue pp = dict(p) pp["stocked"] = _rpm_effective_stocked(p, status_key, assume, mark_delivered) shown.append(pp) if not shown: continue groups.append({ "name": f.name, "wor_type": wor_type, "status": status_key, "title": _wor_display_title(f.name, wor_type, data), "wo_label": _wor_number_label(f.name, data), "part_count": len(parts), "parts": shown, "can_approve": can_approve_by_type.get(wor_type, False), "mtime": f.stat().st_mtime, }) groups.sort(key=lambda g: g["mtime"], reverse=True) for g in groups: del g["mtime"] return jsonify(groups=groups) @app.route("/rpm/wo-parts") @rpm_required def rpm_wo_parts(): """Parts associated with a single Work Order (used by the WO Admin dialog).""" filename = (request.args.get("filename") or "").strip() wor_type = (request.args.get("type") or "").strip() or None sidecar, data = _rpm_load_wo(filename, wor_type) if not sidecar: return jsonify(error="Work order not found"), 404 parts = _rpm_wo_parts(sidecar, data) resolved_type = data.get("_wor_type") or wor_type or "" return jsonify( name=filename, wor_type=resolved_type, title=_wor_display_title(filename, resolved_type, data), wo_label=_wor_number_label(filename, data), parts=parts, can_approve=_rpm_user_can_approve(session.get("user") or {}, resolved_type), ) @app.route("/rpm/manual-search") @rpm_required def rpm_manual_search(): """Search Equipment Manuals for a part from the RPM / WO Admin parts lists (DVI-626). With ``scope=wo`` (default) the search is restricted to the manual attached to the Work Order, if one is provided; ``scope=all`` searches every manual. ``wo_manual`` echoes the attached manual so the client can offer the "Search All Manuals" fallback only when a WO-scoped manual exists.""" q = (request.args.get("q") or "").strip() filename = (request.args.get("filename") or "").strip() wor_type = (request.args.get("type") or "").strip() or None scope = (request.args.get("scope") or "wo").strip() wo_manual = None if filename: _sidecar, data = _rpm_load_wo(filename, wor_type) if data: m = data.get("manual") or {} fn = m.get("filename") if fn and (WOR_MANUALS_DIR / fn).is_file(): wo_manual = {"filename": fn, "original_name": m.get("original_name") or fn} only = wo_manual["filename"] if (scope == "wo" and wo_manual) else None results = _em_search_manuals(q, only_filename=only) if len(q) >= 2 else [] return jsonify(results=results, query=q, scope=scope, wo_manual=wo_manual) @app.route("/rpm/wo-parts/add", methods=["POST"]) @rpm_required def rpm_wo_parts_add(): """Associate a new part with a Work Order and remember its Part Number.""" body = request.get_json(silent=True) or {} filename = (body.get("filename") or "").strip() wor_type = (body.get("type") or "").strip() or None number = (body.get("number") or "").strip() name = (body.get("name") or "").strip() qty = (body.get("qty") or "").strip() amount = (body.get("amount") or "").strip() # DVI-626: "Stocked Item" checkbox on the Add Parts dialog marks the part as # already in stock; the client also disables its Ordered button. stocked = bool(body.get("stocked")) if not (number or name or qty or amount): return jsonify(error="Enter at least one part detail"), 400 sidecar, data = _rpm_load_wo(filename, wor_type) if not sidecar: return jsonify(error="Work order not found"), 404 parts = _rpm_wo_parts(sidecar, data) resolved_type = data.get("_wor_type") or wor_type or "" # DVI-614: auto-approve when the acting user matches the Auto Approval policy. auto_approved = _rpm_auto_approve_for(session.get("user") or {}, resolved_type) new_part = { "id": f"rpm_{uuid.uuid4().hex[:12]}", "number": number, "name": name, "qty": qty, "amount": amount, "approved": auto_approved, "ordered": False, "stocked": stocked, "tracking_number": "", "carrier": "", "tracking_status": "", "delivered_at": "", } parts.append(new_part) _rpm_save_wo_parts(sidecar, data, parts) _rpm_upsert_known_part(number, name) # Notify opted-in users: approved straight away, or newly awaiting approval. _rpm_notify_part( notifications.EVENT_RPM_PART_APPROVED if auto_approved else notifications.EVENT_RPM_PART_AWAITING, resolved_type, filename, data, new_part) if _is_history_capture_enabled("rpm"): _action = "part_approved" if auto_approved else "part_requested" _label = "auto-approved" if auto_approved else "requested" record_history( "rpm", _action, actor=(session.get("user") or {}).get("name"), summary=f"Part {_label}: {number or name or 'unknown'} for WO {filename}", ref=filename, detail={"part_id": new_part["id"], "number": number, "name": name, "wor_type": resolved_type}, ) return jsonify(ok=True, part=new_part) @app.route("/rpm/part/update", methods=["POST"]) @rpm_required def rpm_part_update(): """Edit a part's details in place; re-remembers the Part Number -> Name.""" body = request.get_json(silent=True) or {} filename = (body.get("filename") or "").strip() wor_type = (body.get("type") or "").strip() or None pid = (body.get("id") or "").strip() sidecar, data = _rpm_load_wo(filename, wor_type) if not sidecar: return jsonify(error="Work order not found"), 404 parts = _rpm_wo_parts(sidecar, data) target = next((p for p in parts if p["id"] == pid), None) if not target: return jsonify(error="Part not found"), 404 for key in ("number", "name", "qty", "amount", "tracking_number", "carrier"): if body.get(key) is not None: target[key] = str(body[key]).strip() _rpm_save_wo_parts(sidecar, data, parts) _rpm_upsert_known_part(target["number"], target["name"]) return jsonify(ok=True, part=target) @app.route("/rpm/part/remove", methods=["POST"]) @rpm_required def rpm_part_remove(): """Remove a part from a Work Order.""" body = request.get_json(silent=True) or {} filename = (body.get("filename") or "").strip() wor_type = (body.get("type") or "").strip() or None pid = (body.get("id") or "").strip() sidecar, data = _rpm_load_wo(filename, wor_type) if not sidecar: return jsonify(error="Work order not found"), 404 parts = _rpm_wo_parts(sidecar, data) new_parts = [p for p in parts if p["id"] != pid] if len(new_parts) == len(parts): return jsonify(error="Part not found"), 404 _rpm_save_wo_parts(sidecar, data, new_parts) return jsonify(ok=True) @app.route("/rpm/part/flag", methods=["POST"]) @rpm_required def rpm_part_flag(): """Mark a part approved/ordered/stocked (booleans sent in the body).""" body = request.get_json(silent=True) or {} filename = (body.get("filename") or "").strip() wor_type = (body.get("type") or "").strip() or None pid = (body.get("id") or "").strip() sidecar, data = _rpm_load_wo(filename, wor_type) if not sidecar: return jsonify(error="Work order not found"), 404 parts = _rpm_wo_parts(sidecar, data) target = next((p for p in parts if p["id"] == pid), None) if not target: return jsonify(error="Part not found"), 404 resolved_type = data.get("_wor_type") or wor_type or "" was_approved = bool(target.get("approved")) was_ordered = bool(target.get("ordered")) was_stocked = bool(target.get("stocked")) # DVI-614: only users who meet the Approval permission policy may approve. if "approved" in body and bool(body["approved"]) and not was_approved: if not _rpm_user_can_approve(session.get("user") or {}, resolved_type): return jsonify(error="You do not have permission to approve parts."), 403 if "approved" in body: target["approved"] = bool(body["approved"]) if "ordered" in body: target["ordered"] = bool(body["ordered"]) if "stocked" in body: target["stocked"] = bool(body["stocked"]) # DVI-609: the Mark-Ordered popup may pass an optional tracking number # (and carrier) in the same call that flips ordered=true. if "tracking_number" in body: target["tracking_number"] = str(body["tracking_number"] or "").strip() if "carrier" in body: target["carrier"] = str(body["carrier"] or "").strip() _rpm_save_wo_parts(sidecar, data, parts) # DVI-614: fire part-lifecycle emails on off->on transitions only. if target.get("approved") and not was_approved: _rpm_notify_part(notifications.EVENT_RPM_PART_APPROVED, resolved_type, filename, data, target) if target.get("ordered") and not was_ordered: _rpm_notify_part(notifications.EVENT_RPM_PART_ORDERED, resolved_type, filename, data, target) if target.get("stocked") and not was_stocked: _rpm_notify_part(notifications.EVENT_RPM_PART_STOCKED, resolved_type, filename, data, target) if _is_history_capture_enabled("rpm"): _part_label = target.get("number") or target.get("name") or target.get("id", "") if target.get("approved") and not was_approved: record_history( "rpm", "part_approved", actor=(session.get("user") or {}).get("name"), summary=f"Part approved: {_part_label} for WO {filename}", ref=filename, detail={"part_id": pid, "number": target.get("number"), "wor_type": resolved_type}, ) if target.get("ordered") and not was_ordered: record_history( "rpm", "part_ordered", actor=(session.get("user") or {}).get("name"), summary=f"Part ordered: {_part_label} for WO {filename}", ref=filename, detail={"part_id": pid, "number": target.get("number"), "tracking": target.get("tracking_number"), "wor_type": resolved_type}, ) if target.get("stocked") and not was_stocked: record_history( "rpm", "part_stocked", actor=(session.get("user") or {}).get("name"), summary=f"Part stocked: {_part_label} for WO {filename}", ref=filename, detail={"part_id": pid, "number": target.get("number"), "wor_type": resolved_type}, ) return jsonify(ok=True, part=target) @app.route("/rpm/known-part") @rpm_required def rpm_known_part(): """Autofill lookup: return the remembered Part Name for a Part Number.""" number = (request.args.get("number") or "").strip() if not number: return jsonify(name="") return jsonify(name=_rpm_load_known_parts().get(number, "")) @app.route("/rpm/settings") @rpm_required def rpm_settings_get(): """Return the RPM policy settings (Settings tab -> Policy pane).""" return jsonify(settings=_rpm_load_settings()) @app.route("/rpm/settings", methods=["POST"]) @rpm_required def rpm_settings_post(): """Persist RPM policy settings. Only known keys are accepted.""" body = request.get_json(silent=True) or {} settings = _rpm_load_settings() for key in _RPM_SETTINGS_DEFAULTS: if key in body: settings[key] = bool(body[key]) _rpm_save_settings(settings) return jsonify(ok=True, settings=settings) @app.route("/rpm/approval-policy") @rpm_required def rpm_approval_policy_get(): """Return the Approval permission + Auto Approval policy (DVI-614).""" return jsonify(policy=_rpm_load_approval()) @app.route("/rpm/approval-policy", methods=["POST"]) @rpm_required def rpm_approval_policy_post(): """Persist the Approval permission + Auto Approval policy (DVI-614).""" body = request.get_json(silent=True) or {} pol = body.get("policy") if not isinstance(pol, dict): return jsonify(error="Invalid policy."), 400 # Re-validate/normalize through the loader's shape by round-tripping the # supplied values, then save the clean result. clean = { "approve_mode": pol.get("approve_mode") if pol.get("approve_mode") in _RPM_APPROVE_MODES else "all_admins", "approve_users": _rpm_norm_emails(pol.get("approve_users")), "auto_approval": {"enabled": False, "mobilemachine": {"mode": "approvers", "users": []}, "preventive": {"mode": "approvers", "users": []}}, "policy_notify": {wt: [] for wt in RPM_PART_WOR_TYPES}, } auto = pol.get("auto_approval") if isinstance(auto, dict): clean["auto_approval"]["enabled"] = bool(auto.get("enabled")) for wt in RPM_PART_WOR_TYPES: blk = auto.get(wt) if isinstance(blk, dict): if blk.get("mode") in _RPM_AUTO_MODES: clean["auto_approval"][wt]["mode"] = blk["mode"] clean["auto_approval"][wt]["users"] = _rpm_norm_emails(blk.get("users")) pn = pol.get("policy_notify") if isinstance(pn, dict): for wt in RPM_PART_WOR_TYPES: clean["policy_notify"][wt] = _rpm_norm_emails(pn.get(wt)) try: _rpm_save_approval(clean) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True, policy=_rpm_load_approval()) # RPM per-user notification grid (DVI-614). Columns mirror RPM_EVENT_COL. _RPM_NOTIFY_COLS = ("awaiting", "approved", "ordered", "delivered", "stocked") def _rpm_notify_grid_from_settings(settings: dict) -> dict: """Return the {wor_type: {col: bool}} RPM notifications grid, all-off default.""" stored = settings.get(notifications.RPM_NOTIFY_KEY) grid = {} for wt in RPM_PART_WOR_TYPES: src = stored.get(wt) if isinstance(stored, dict) else None src = src if isinstance(src, dict) else {} grid[wt] = {col: bool(src.get(col)) for col in _RPM_NOTIFY_COLS} return grid @app.route("/rpm/notifications") @login_required def rpm_notifications_get(): """Return the current user's RPM notifications grid (DVI-614). Available to all users — the grid is mirrored into User Settings > Notifications.""" email = (session.get("user") or {}).get("email", "") return jsonify(notifications=_rpm_notify_grid_from_settings(_load_user_settings(email))) @app.route("/rpm/notifications", methods=["POST"]) @login_required def rpm_notifications_post(): """Persist the current user's RPM notifications grid (DVI-614).""" body = request.get_json(silent=True) or {} grid_in = body.get("notifications") if not isinstance(grid_in, dict): return jsonify(error="Invalid notifications grid."), 400 email = (session.get("user") or {}).get("email", "") settings = _load_user_settings(email) grid = {} for wt in RPM_PART_WOR_TYPES: src = grid_in.get(wt) if isinstance(grid_in.get(wt), dict) else {} grid[wt] = {col: bool(src.get(col)) for col in _RPM_NOTIFY_COLS} settings[notifications.RPM_NOTIFY_KEY] = grid if email: settings["email"] = email try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True, notifications=grid) @app.route("/rpm/processing") @rpm_required def rpm_processing(): """Processing tab: parts that still need action across active Work Orders. Returns awaiting_approval (not yet approved) and ordered_not_stocked (ordered but not yet stocked). Parts that are effectively stocked (explicitly, or via the assume-stocked-on-complete policy) drop out of both lists. Cancelled WOs are skipped.""" settings = _rpm_load_settings() assume = settings["assume_stocked_on_complete"] mark_delivered = settings["mark_stocked_on_delivered"] awaiting_approval, ordered_not_stocked = [], [] for wor_type in RPM_PART_WOR_TYPES: for status_key, directory in _wor_dest_dirs(wor_type).items(): if status_key == "cancelled" or not directory.is_dir(): continue for f in directory.glob("*.pdf"): sidecar = f.with_suffix(".json") if not sidecar.is_file(): continue try: data = json.loads(sidecar.read_text()) or {} except (ValueError, OSError): continue parts = _rpm_wo_parts(sidecar, data) if not parts: continue for p in parts: if _rpm_effective_stocked(p, status_key, assume, mark_delivered): continue item = { "name": f.name, "wor_type": wor_type, "status": status_key, "title": _wor_display_title(f.name, wor_type, data), "wo_label": _wor_number_label(f.name, data), "part": p, "mtime": f.stat().st_mtime, } if not p.get("approved"): awaiting_approval.append(item) if p.get("ordered"): ordered_not_stocked.append(item) for lst in (awaiting_approval, ordered_not_stocked): lst.sort(key=lambda it: it["mtime"], reverse=True) for it in lst: del it["mtime"] return jsonify(awaiting_approval=awaiting_approval, ordered_not_stocked=ordered_not_stocked) @app.route("/wor-admin-status", methods=["POST"]) @wor_admin_required def wor_admin_status(): """Change a WOR's status by moving it between directories.""" body = request.get_json(silent=True) or {} filename = body.get("filename", "") new_status = body.get("status", "") if not filename or ".." in filename or "/" in filename: return jsonify(error="Invalid filename"), 400 if new_status not in ("new", "in_progress", "completed", "cancelled"): return jsonify(error="Invalid status"), 400 src_path = _resolve_wor_file(filename) if not src_path: return jsonify(error="File not found"), 404 wor_type = _detect_wor_type(src_path) # When moving to in_progress for maintenance WOs, fill maintenance fields and regenerate PDF if new_status == "in_progress" and wor_type == "maintenance": sidecar_path = _resolve_wor_sidecar(filename) if sidecar_path: try: form_data = json.loads(sidecar_path.read_text()) except (json.JSONDecodeError, OSError): form_data = {} else: form_data = {} user = session.get("user", {}) maintenance = form_data.get("_maintenance", {}) if not maintenance.get("accepted_by"): maintenance["accepted_by"] = user.get("name", "") if not maintenance.get("date"): maintenance["date"] = _datetime_type.now().strftime("%Y-%m-%d") if not maintenance.get("work_order_number"): maintenance["work_order_number"] = str(_next_work_order_number()) form_data["_maintenance"] = maintenance pdf_bytes = _generate_wor_pdf(form_data, maintenance=maintenance) src_path.write_bytes(pdf_bytes) if sidecar_path: sidecar_path.write_text(json.dumps(form_data, indent=2)) # When moving to in_progress for preventive maintenance WOs, fill maintenance # acceptance fields and regenerate PDF. if new_status == "in_progress" and wor_type == "preventive": sidecar_path = _resolve_wor_sidecar(filename) if sidecar_path: try: form_data = json.loads(sidecar_path.read_text()) except (json.JSONDecodeError, OSError): form_data = {} else: form_data = {} user = session.get("user", {}) maintenance = form_data.get("_maintenance", {}) if not maintenance.get("accepted_by"): maintenance["accepted_by"] = user.get("name", "") if not maintenance.get("date"): maintenance["date"] = _datetime_type.now().strftime("%Y-%m-%d") if not maintenance.get("work_order_number"): maintenance["work_order_number"] = str(_next_work_order_number()) form_data["_maintenance"] = maintenance pdf_bytes = _generate_preventive_wor_pdf(form_data, maintenance=maintenance) src_path.write_bytes(pdf_bytes) if sidecar_path: sidecar_path.write_text(json.dumps(form_data, indent=2)) # When moving to in_progress for safety WOs, fill safety acceptance fields and regenerate PDF if new_status == "in_progress" and wor_type == "safety": sidecar_path = _resolve_wor_sidecar(filename) if sidecar_path: try: form_data = json.loads(sidecar_path.read_text()) except (json.JSONDecodeError, OSError): form_data = {} else: form_data = {} user = session.get("user", {}) safety = form_data.get("_safety", {}) if not safety.get("accepted_by"): safety["accepted_by"] = user.get("name", "") if not safety.get("date"): safety["date"] = _datetime_type.now().strftime("%Y-%m-%d") if not safety.get("work_order_number"): safety["work_order_number"] = str(_next_work_order_number()) form_data["_safety"] = safety pdf_bytes = _generate_safety_wor_pdf(form_data, safety=safety) src_path.write_bytes(pdf_bytes) if sidecar_path: sidecar_path.write_text(json.dumps(form_data, indent=2)) # When moving to in_progress for mobile machine WOs, assign order number and regenerate PDF if new_status == "in_progress" and wor_type == "mobilemachine": sidecar_path = _resolve_wor_sidecar(filename) if sidecar_path: try: form_data = json.loads(sidecar_path.read_text()) except (json.JSONDecodeError, OSError): form_data = {} else: form_data = {} if not form_data.get("_order_number"): form_data["_order_number"] = str(_next_work_order_number()) pdf_bytes = _generate_mm_wor_pdf(form_data) src_path.write_bytes(pdf_bytes) if sidecar_path: sidecar_path.write_text(json.dumps(form_data, indent=2)) # Locate the sidecar before moving anything cur_sidecar = _resolve_wor_sidecar(filename) dest_dir = _wor_dest_dirs(wor_type)[new_status] dest_dir.mkdir(parents=True, exist_ok=True) dest_path = dest_dir / filename if src_path.resolve() != dest_path.resolve(): shutil.move(str(src_path), str(dest_path)) # Move JSON sidecar alongside the PDF dest_sidecar = None if cur_sidecar: dest_sidecar = dest_path.with_suffix(".json") if cur_sidecar.resolve() != dest_sidecar.resolve(): shutil.move(str(cur_sidecar), str(dest_sidecar)) # Raise the matching lifecycle event (one branded email per recipient). # "new" (reopen/back-to-new) is not a notifiable transition. event = _WOR_STATUS_EVENTS.get(new_status) if event: sidecar_data = {} if dest_sidecar and dest_sidecar.is_file(): try: sidecar_data = json.loads(dest_sidecar.read_text()) except (json.JSONDecodeError, OSError): sidecar_data = {} notifications.notify(event, _wor_event_context(filename, wor_type, sidecar_data)) if _is_history_capture_enabled("wor-admin"): _status_label = { "new": "reopened", "in_progress": "accepted", "completed": "completed", "cancelled": "cancelled", }.get(new_status, new_status) record_history( "wor-admin", "status_changed", actor=(session.get("user") or {}).get("name"), summary=f"WO {filename} {_status_label}", ref=filename, detail={"wor_type": wor_type, "new_status": new_status}, ) return jsonify(ok=True, status=new_status) @app.route("/wor-admin-data") @wor_admin_required def wor_admin_data(): """Return JSON sidecar data for a WOR so the admin can populate the form.""" filename = request.args.get("file", "") sidecar = _resolve_wor_sidecar(filename) if sidecar: try: data = json.loads(sidecar.read_text()) return jsonify(data=data) except (json.JSONDecodeError, OSError): pass # Fallback: extract form data from the PDF itself pdf_path = _resolve_wor_file(filename) if not pdf_path: return jsonify(error="Work order not found."), 404 try: data = _extract_wor_data_from_pdf(pdf_path) return jsonify(data=data) except Exception as exc: return jsonify(error=f"Could not extract form data: {exc}"), 500 @app.route("/wor-edit", methods=["POST"]) @wor_admin_required def wor_edit(): """Admin-only: update an existing WOR with edited form data and regenerate PDF.""" edit_filename = request.form.get("edit_filename", "").strip() if not edit_filename or ".." in edit_filename or "/" in edit_filename: return jsonify(error="Invalid filename"), 400 pdf_path = _resolve_wor_file(edit_filename) if not pdf_path: return jsonify(error="Original work order not found"), 404 data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 # Preserve existing maintenance fields from sidecar sidecar_path = _resolve_wor_sidecar(edit_filename) maintenance = {} if sidecar_path: try: old_data = json.loads(sidecar_path.read_text()) maintenance = old_data.get("_maintenance", {}) except (json.JSONDecodeError, OSError): pass data["_wor_type"] = "maintenance" data["_maintenance"] = maintenance pdf_bytes = _generate_wor_pdf(data, maintenance=maintenance or None) pdf_path.write_bytes(pdf_bytes) # Update sidecar in the same directory as the PDF json_path = pdf_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "edited", actor=(session.get("user") or {}).get("name"), summary=f"Maintenance WO edited — {data.get('requestor_name', edit_filename)}", ref=edit_filename, detail={"wor_type": "maintenance"}, ) return jsonify(filename=edit_filename) @app.route("/safety-wor-edit", methods=["POST"]) @wor_admin_required def safety_wor_edit(): """Admin-only: update an existing Safety WOR with edited form data and regenerate PDF.""" edit_filename = request.form.get("edit_filename", "").strip() if not edit_filename or ".." in edit_filename or "/" in edit_filename: return jsonify(error="Invalid filename"), 400 pdf_path = _resolve_wor_file(edit_filename, wor_type="safety") if not pdf_path: return jsonify(error="Original work order not found"), 404 data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 # Preserve existing safety acceptance fields from sidecar sidecar_path = _resolve_wor_sidecar(edit_filename, wor_type="safety") safety = {} if sidecar_path: try: old_data = json.loads(sidecar_path.read_text()) safety = old_data.get("_safety", {}) except (json.JSONDecodeError, OSError): pass data["_wor_type"] = "safety" data["_safety"] = safety pdf_bytes = _generate_safety_wor_pdf(data, safety=safety or None) pdf_path.write_bytes(pdf_bytes) # Update sidecar in the same directory as the PDF json_path = pdf_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "edited", actor=(session.get("user") or {}).get("name"), summary=f"Safety WO edited — {data.get('requestor_name', edit_filename)}", ref=edit_filename, detail={"wor_type": "safety"}, ) return jsonify(filename=edit_filename) @app.route("/mm-wor-edit", methods=["POST"]) @wor_admin_required def mm_wor_edit(): """Admin-only: update an existing Mobile Machine WOR and regenerate PDF.""" edit_filename = request.form.get("edit_filename", "").strip() if not edit_filename or ".." in edit_filename or "/" in edit_filename: return jsonify(error="Invalid filename"), 400 pdf_path = _resolve_wor_file(edit_filename, wor_type="mobilemachine") if not pdf_path: return jsonify(error="Original work order not found"), 404 part_numbers = request.form.getlist("part_number[]") part_names = request.form.getlist("part_name[]") part_qtys = request.form.getlist("part_qty[]") part_amounts = request.form.getlist("part_amount[]") parts = [] for i in range(len(part_numbers)): pn = part_numbers[i].strip() if i < len(part_numbers) else "" nm = part_names[i].strip() if i < len(part_names) else "" qt = part_qtys[i].strip() if i < len(part_qtys) else "" am = part_amounts[i].strip() if i < len(part_amounts) else "" if pn or nm or qt or am: parts.append({"number": pn, "name": nm, "qty": qt, "amount": am}) data = { "machine": request.form.get("machine", "").strip(), "asset_number": request.form.get("asset_number", "").strip(), "order_date_time": request.form.get("order_date_time", "").strip(), "date_promised": request.form.get("date_promised", "").strip(), "date_delivered": request.form.get("date_delivered", "").strip(), "order_received_by": request.form.get("order_received_by", "").strip(), "make_model": request.form.get("make_model", "").strip(), "vin": request.form.get("vin", "").strip(), "odometer": request.form.get("odometer", "").strip(), "license_state": request.form.get("license_state", "").strip(), "motor_number": request.form.get("motor_number", "").strip(), "service_type": request.form.getlist("service_type"), "labor_description": request.form.get("labor_description", "").strip(), "labor_time": request.form.get("labor_time", "").strip(), "labor_total": request.form.get("labor_total", "").strip(), "parts": parts, "awaiting_approval": "awaiting_approval" in request.form, "parts_ordered": "parts_ordered" in request.form, "parts_total": request.form.get("parts_total", "").strip(), "compiled_by": request.form.get("compiled_by", "").strip(), "compiled_date": request.form.get("compiled_date", "").strip(), "authorized_by": request.form.get("authorized_by", "").strip(), "authorization_date": request.form.get("authorization_date", "").strip(), "_wor_type": "mobilemachine", } if not data["machine"]: return jsonify(error="Machine name is required"), 400 sidecar_path = _resolve_wor_sidecar(edit_filename, wor_type="mobilemachine") if sidecar_path: try: old_data = json.loads(sidecar_path.read_text()) if old_data.get("_order_number"): data["_order_number"] = old_data["_order_number"] # Preserve a previously-attached manual unless this edit replaces it. if old_data.get("manual"): data["manual"] = old_data["manual"] except (json.JSONDecodeError, OSError): pass manual_info, manual_err = _pm_process_manual(data["machine"], session.get("user") or {}) if manual_err: return jsonify(error=manual_err), 400 if manual_info: data["manual"] = manual_info pdf_bytes = _generate_mm_wor_pdf(data) pdf_path.write_bytes(pdf_bytes) json_path = pdf_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "edited", actor=(session.get("user") or {}).get("name"), summary=f"Mobile Machine WO edited — {data.get('machine', edit_filename)}", ref=edit_filename, detail={"wor_type": "mobilemachine"}, ) return jsonify(filename=edit_filename) @app.route("/preventive-wor-edit", methods=["POST"]) @wor_admin_required def preventive_wor_edit(): """Admin-only: update an existing Preventative Maintenance WOR and regenerate PDF.""" edit_filename = request.form.get("edit_filename", "").strip() if not edit_filename or ".." in edit_filename or "/" in edit_filename: return jsonify(error="Invalid filename"), 400 pdf_path = _resolve_wor_file(edit_filename, wor_type="preventive") if not pdf_path: return jsonify(error="Original work order not found"), 404 part_numbers = request.form.getlist("part_number[]") part_names = request.form.getlist("part_name[]") part_qtys = request.form.getlist("part_qty[]") part_amounts = request.form.getlist("part_amount[]") parts = [] for i in range(len(part_numbers)): pn = part_numbers[i].strip() if i < len(part_numbers) else "" nm = part_names[i].strip() if i < len(part_names) else "" qt = part_qtys[i].strip() if i < len(part_qtys) else "" am = part_amounts[i].strip() if i < len(part_amounts) else "" if pn or nm or qt or am: parts.append({"number": pn, "name": nm, "qty": qt, "amount": am}) data = { "requestor_name": request.form.get("requestor_name", "").strip(), "phone": request.form.get("phone", "").strip(), "email": request.form.get("email", "").strip(), "date_of_request": request.form.get("date_of_request", "").strip(), "location": request.form.get("location", "").strip(), "building": request.form.get("building", "").strip(), "man_hole": request.form.get("man_hole", "").strip(), "equipment": request.form.get("equipment", "").strip(), "description": request.form.get("description", "").strip(), "issue_type": request.form.getlist("issue_type"), "issue_type_other": request.form.get("issue_type_other", "").strip(), "best_times": request.form.get("best_times", "").strip(), "desired_completion_date": request.form.get("desired_completion_date", "").strip(), "urgency": request.form.get("urgency", "").strip(), "parts": parts, "awaiting_approval": "awaiting_approval" in request.form, "parts_ordered": "parts_ordered" in request.form, "parts_total": request.form.get("parts_total", "").strip(), } if not data["requestor_name"]: return jsonify(error="Requestor name is required"), 400 if not data["description"]: return jsonify(error="Request description is required"), 400 sidecar_path = _resolve_wor_sidecar(edit_filename, wor_type="preventive") maintenance = {} if sidecar_path: try: old_data = json.loads(sidecar_path.read_text()) maintenance = old_data.get("_maintenance", {}) # Preserve a previously-attached manual unless this edit replaces it. if old_data.get("manual"): data["manual"] = old_data["manual"] except (json.JSONDecodeError, OSError): pass manual_info, manual_err = _pm_process_manual(data["equipment"], session.get("user") or {}) if manual_err: return jsonify(error=manual_err), 400 if manual_info: data["manual"] = manual_info data["_wor_type"] = "preventive" data["_maintenance"] = maintenance pdf_bytes = _generate_preventive_wor_pdf(data, maintenance=maintenance or None) pdf_path.write_bytes(pdf_bytes) json_path = pdf_path.with_suffix(".json") json_path.write_text(json.dumps(data, indent=2)) if _is_history_capture_enabled("wor-admin"): record_history( "wor-admin", "edited", actor=(session.get("user") or {}).get("name"), summary=f"Preventive Maint. WO edited — {data.get('requestor_name', edit_filename)}", ref=edit_filename, detail={"wor_type": "preventive"}, ) return jsonify(filename=edit_filename, manual=data.get("manual")) # --------------------------------------------------------------------------- # Routes — Recurring Work Orders # --------------------------------------------------------------------------- @app.route("/wor-make-recurring", methods=["POST"]) @wor_admin_required def wor_make_recurring(): """Mark a work order as recurring and register it in the recurring registry.""" body = request.get_json(silent=True) or {} filename = body.get("filename", "") frequency_days = body.get("frequency_days") start_date = body.get("start_date", "") end_date = body.get("end_date", "") if not filename or ".." in filename or "/" in filename: return jsonify(error="Invalid filename"), 400 try: frequency_days = int(frequency_days) if frequency_days < 1: raise ValueError except (TypeError, ValueError): return jsonify(error="Frequency must be a positive number of days"), 400 if not start_date: start_date = _datetime_type.now().strftime("%Y-%m-%d") pdf_path = _resolve_wor_file(filename) if not pdf_path: return jsonify(error="Work order not found"), 404 wor_type = _detect_wor_type(pdf_path) sidecar = _resolve_wor_sidecar(filename) if not sidecar: return jsonify(error="Work order sidecar data not found"), 404 try: form_data = json.loads(sidecar.read_text()) except (json.JSONDecodeError, OSError): return jsonify(error="Could not read work order data"), 500 next_run = _calc_next_run(start_date, frequency_days) recurring_config = { "frequency_days": frequency_days, "start_date": start_date, "end_date": end_date, "next_run": next_run, "wor_type": wor_type, "source_filename": filename, "enabled": True, "created_by": session.get("user", {}).get("name", ""), "created_at": _datetime_type.now().strftime("%Y-%m-%dT%H:%M:%S"), } form_data["_recurring"] = recurring_config try: sidecar.write_text(json.dumps(form_data, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save recurring config: {exc}"), 500 registry = _load_recurring_registry() registry[filename] = recurring_config registry[filename]["template_data"] = {k: v for k, v in form_data.items() if k != "_recurring"} try: _save_recurring_registry(registry) except (OSError, ValueError, TypeError) as exc: return jsonify(error=f"Failed to update recurring registry: {exc}"), 500 return jsonify(ok=True, next_run=next_run, recurring=recurring_config) @app.route("/wor-remove-recurring", methods=["POST"]) @wor_admin_required def wor_remove_recurring(): """Remove recurring status from a work order.""" body = request.get_json(silent=True) or {} filename = body.get("filename", "") if not filename or ".." in filename or "/" in filename: return jsonify(error="Invalid filename"), 400 sidecar = _resolve_wor_sidecar(filename) if sidecar: try: form_data = json.loads(sidecar.read_text()) form_data.pop("_recurring", None) sidecar.write_text(json.dumps(form_data, indent=2)) except (json.JSONDecodeError, OSError): pass registry = _load_recurring_registry() registry.pop(filename, None) try: _save_recurring_registry(registry) except OSError as exc: return jsonify(error=f"Failed to update recurring registry: {exc}"), 500 return jsonify(ok=True) @app.route("/wor-recurring-run", methods=["POST"]) @wor_admin_required def wor_recurring_run(): """Manually trigger processing of due recurring work orders.""" created = _process_recurring_work_orders() return jsonify(ok=True, created=created) def _process_recurring_work_orders() -> list[str]: """Check registry for due recurring WOs and create new instances. Returns list of created filenames.""" from datetime import timedelta registry = _load_recurring_registry() today = _datetime_type.now().strftime("%Y-%m-%d") created = [] changed = False for source_filename, entry in list(registry.items()): if not entry.get("enabled", True): continue end_date = entry.get("end_date", "") if end_date and end_date < today: entry["enabled"] = False changed = True continue next_run = entry.get("next_run", "") if not next_run or next_run > today: continue template = entry.get("template_data", {}) if not template: continue wor_type = entry.get("wor_type", template.get("_wor_type", "maintenance")) template_clean = {k: v for k, v in template.items() if not k.startswith("_")} wor_num = _next_work_order_number() if wor_type == "maintenance": template_clean["date_of_request"] = today template_clean["_wor_type"] = "maintenance" template_clean["_wor_number"] = wor_num WOR_MAINTENANCE_DIR.mkdir(parents=True, exist_ok=True) pdf_bytes = _generate_wor_pdf(template_clean) safe_name = re.sub(r"[^\w\s\-]", "", template_clean.get("requestor_name", "Recurring"))[:40].strip().replace(" ", "_") timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") new_filename = f"WOR_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_MAINTENANCE_DIR / new_filename out_path.write_bytes(pdf_bytes) template_clean["_recurring_source"] = source_filename out_path.with_suffix(".json").write_text(json.dumps(template_clean, indent=2)) elif wor_type == "safety": template_clean["date_of_request"] = today template_clean["_wor_type"] = "safety" template_clean["_wor_number"] = wor_num WOR_SAFETY_DIR.mkdir(parents=True, exist_ok=True) pdf_bytes = _generate_safety_wor_pdf(template_clean) safe_name = re.sub(r"[^\w\s\-]", "", template_clean.get("requestor_name", "Recurring"))[:40].strip().replace(" ", "_") timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") new_filename = f"SWO_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_SAFETY_DIR / new_filename out_path.write_bytes(pdf_bytes) template_clean["_recurring_source"] = source_filename out_path.with_suffix(".json").write_text(json.dumps(template_clean, indent=2)) elif wor_type == "preventive": template_clean["date_of_request"] = today template_clean["_wor_type"] = "preventive" template_clean["_wor_number"] = wor_num WOR_PREVENTIVE_DIR.mkdir(parents=True, exist_ok=True) pdf_bytes = _generate_preventive_wor_pdf(template_clean) safe_name = re.sub(r"[^\w\s\-]", "", template_clean.get("requestor_name", "Recurring"))[:40].strip().replace(" ", "_") timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") new_filename = f"PMWO_{wor_num:05d}_{timestamp}_{safe_name}.pdf" out_path = WOR_PREVENTIVE_DIR / new_filename out_path.write_bytes(pdf_bytes) template_clean["_recurring_source"] = source_filename out_path.with_suffix(".json").write_text(json.dumps(template_clean, indent=2)) else: template_clean["order_date_time"] = _datetime_type.now().strftime("%Y-%m-%dT%H:%M") template_clean["_wor_type"] = "mobilemachine" template_clean["_wor_number"] = wor_num WOR_MOBILEMACHINE_DIR.mkdir(parents=True, exist_ok=True) pdf_bytes = _generate_mm_wor_pdf(template_clean) safe_machine = re.sub(r"[^\w\s\-]", "", template_clean.get("machine", "Recurring"))[:40].strip().replace(" ", "_") timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") new_filename = f"MMWO_{wor_num:05d}_{timestamp}_{safe_machine}.pdf" out_path = WOR_MOBILEMACHINE_DIR / new_filename out_path.write_bytes(pdf_bytes) template_clean["_recurring_source"] = source_filename out_path.with_suffix(".json").write_text(json.dumps(template_clean, indent=2)) created.append(new_filename) freq = entry.get("frequency_days", 90) entry["next_run"] = _calc_next_run(entry.get("start_date", today), freq, after_date=today) entry["last_created"] = new_filename entry["last_run"] = today changed = True if changed: _save_recurring_registry(registry) return created # --------------------------------------------------------------------------- # Route — create carrier sheet from scratch (DVI-305) # --------------------------------------------------------------------------- _CARRIER_FIELDS = [ "job_number", "part_ref", "product_code", "weight", "production_date", "structure_id", "spec_type", "structure_size", "station", "customer_name", "project_name", ] def _run_create_carrier(job_folder: Path, shop_path: Path, submittal_path: "Path | None", fields: dict) -> None: """Background thread: build carrier sheet from scratch then merge QR codes.""" try: from qr_merge import create_carrier_sheet # noqa: E402 — implemented by Linux Automation Engineer (DVI-305) except ImportError: (job_folder / "error.txt").write_text( "create_carrier_sheet not yet available in qr_merge.py. " "Waiting on Linux Automation Engineer (DVI-305).\n" ) return # Step 1: generate carrier sheet PDF with text fields populated carrier_path = create_carrier_sheet(fields=fields, output_folder=job_folder) # Step 2: merge QR codes onto the generated carrier sheet folder_name = job_folder.name folder_url = f"{_get_qr_base_url().rstrip('/')}/{quote(folder_name)}" shop_url = f"{folder_url}/{quote(shop_path.name)}" submittal_url = ( f"{folder_url}/{quote(submittal_path.name)}" if submittal_path else None ) blank_bytes = carrier_path.read_bytes() result_bytes = build_carrier_pdf(blank_bytes, submittal_url, shop_url) output_path = job_folder / f"{job_folder.name} CarrierQR generated.pdf" output_path.write_bytes(result_bytes) carrier_path.unlink(missing_ok=True) # remove intermediate; keep final generated @app.route("/create-carrier", methods=["POST"]) @login_required def create_carrier(): """Accept shop/submittal PDFs + carrier field data, create carrier sheet from scratch.""" shop = request.files.get("shop") submittal = request.files.get("submittal") if not (shop and shop.filename): return jsonify(error="Shop Drawing PDF is required"), 400 if not shop.filename.lower().endswith(".pdf"): return jsonify(error="'shop' must be a PDF file"), 400 for req_field in ("job_number", "product_code", "customer_name", "project_name"): if not request.form.get(req_field, "").strip(): return jsonify(error=f"'{req_field}' is required"), 400 fields = {f: request.form.get(f, "").strip() for f in _CARRIER_FIELDS} job_name = request.form.get("job_name", "").strip() job_id = ( re.sub(r"[^\w\s\-\.]", "_", job_name)[:80].strip() if job_name else str(uuid.uuid4())[:8] ) job_folder = _safe_job_folder(job_id) if _find_generated(job_folder): return jsonify(job_id=job_id) job_folder.mkdir(parents=True, exist_ok=True) shop_path = job_folder / secure_filename(shop.filename) shop.save(shop_path) submittal_path: "Path | None" = None if submittal and submittal.filename and submittal.filename.lower().endswith(".pdf"): submittal_path = job_folder / secure_filename(submittal.filename) submittal.save(submittal_path) t = threading.Thread( target=_run_create_carrier, args=(job_folder, shop_path, submittal_path, fields), daemon=True, ) t.start() if _is_history_capture_enabled("single-carrier-sheet"): record_history( "single-carrier-sheet", "generated", actor=(session.get("user") or {}).get("name"), summary=( f"Carrier sheet generated — {fields.get('customer_name', '')} / " f"{fields.get('project_name', '')} ({fields.get('product_code', '')})" ), ref=job_id, detail={ "customer": fields.get("customer_name"), "project": fields.get("project_name"), "vendor_code": fields.get("vendor_project_code"), "product_code": fields.get("product_code"), }, ) return jsonify(job_id=job_id) def _suggest_fields_from_pdf(pdf_path: Path, allowed: list[str]) -> dict: """Best-effort title-block extraction, filtered to `allowed` Create fields. Returns {} on any failure — OCR prefill is a convenience and must never block the Create flow. Honors the never-add-CSV-columns rule by only ever suggesting values for fields that already exist on the form. """ try: from ocr_engine import extract_title_block_fields suggestions = extract_title_block_fields(pdf_path) except Exception as exc: # noqa: BLE001 app.logger.warning("title-block extraction failed for %s: %s", pdf_path, exc) return {} return {k: v for k, v in suggestions.items() if k in allowed and v} @app.route("/create-carrier/extract", methods=["POST"]) @takeoff_workflow_required def create_carrier_extract(): """OCR a just-selected drawing's title block (DVI-498) and return suggested, user-editable Create-Carrier field values. The upload is read in a temp dir and discarded — nothing is persisted and the take-off CSV is untouched.""" f = request.files.get("file") or request.files.get("shop") if not (f and f.filename): return jsonify(error="A PDF file is required"), 400 if not f.filename.lower().endswith(".pdf"): return jsonify(error="File must be a PDF"), 400 with tempfile.TemporaryDirectory() as td: tmp = Path(td) / secure_filename(f.filename) f.save(str(tmp)) fields = _suggest_fields_from_pdf(tmp, _CARRIER_FIELDS) return jsonify(fields=fields) # Meta-modal fields the Take-off Bridge project-info popup can be prefilled with. _META_FIELDS = [ "customer_name", "project_name", "job_number", "production_date", "spec_type", "station", ] @app.route("/takeoff/extract-metadata", methods=["POST"]) @takeoff_workflow_required def takeoff_extract_metadata(): """OCR a representative drawing in a published folder (DVI-498) to suggest project-info values for the Take-off Bridge metadata popup. Prefers a carrier sheet (metadata sits below the QR zone), else a shop/submittal drawing.""" data = request.get_json(silent=True) or {} folder_name = data.get("folder", "") if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify(error="Invalid folder"), 400 folder = WATCH_DIR / folder_name if not folder.exists(): return jsonify(error="Folder not found"), 400 state = _load_state(folder) files = state.get("files", {}) or {} candidate: "Path | None" = None for ftype in ("carrier", "shop", "submittal"): for item in files.values(): name = (item or {}).get(ftype) if not name: continue p = (folder / name).resolve() if str(p).startswith(str(folder.resolve())) and p.is_file() \ and p.suffix.lower() == ".pdf": candidate = p break if candidate: break if candidate is None: return jsonify(fields={}) return jsonify(fields=_suggest_fields_from_pdf(candidate, _META_FIELDS)) def _takeoff_resolve_file(folder_name: str, rel: str) -> "Path | None": """Resolve folder/rel under WATCH_DIR, rejecting traversal. Returns the file Path or None (shared by the DVI-641 project-file preview endpoints).""" if not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name or ".." in rel: return None folder = (WATCH_DIR / folder_name).resolve() file_path = (folder / rel).resolve() if not str(file_path).startswith(str(folder) + os.sep): return None if not file_path.is_file(): return None return file_path @app.route("/takeoff/extract-metadata-file", methods=["POST"]) @takeoff_workflow_required def takeoff_extract_metadata_file(): """OCR the ONE document the user has open in the preview (DVI-641) and suggest project-info values. Unlike /takeoff/extract-metadata (which auto-picks a representative drawing), this reads the exact selected file. PDF-only — image/text documents have no title block to read.""" data = request.get_json(silent=True) or {} file_path = _takeoff_resolve_file(data.get("folder", ""), data.get("file", "")) if file_path is None: return jsonify(error="Invalid path"), 400 if file_path.suffix.lower() != ".pdf": return jsonify(error="OCR can only read PDF documents."), 400 return jsonify(fields=_suggest_fields_from_pdf(file_path, _META_FIELDS)) @app.route("/takeoff/pdf-pages") @takeoff_workflow_required def takeoff_pdf_pages(): """Return a project PDF's page count so the preview can render each page as a fit-to-width image (DVI-641 — native PDF viewers zoom too far in on mobile and offer no controls).""" file_path = _takeoff_resolve_file(request.args.get("folder", ""), request.args.get("file", "")) if file_path is None: return jsonify(error="Invalid path"), 400 if file_path.suffix.lower() != ".pdf": return jsonify(error="Not a PDF"), 400 try: import fitz doc = fitz.open(str(file_path)) try: pages = doc.page_count finally: doc.close() except Exception as exc: # noqa: BLE001 — surface to UI for iframe fallback return jsonify(error=f"Could not read PDF: {exc}"), 500 return jsonify(pages=pages) @app.route("/takeoff/pdf-page") @takeoff_workflow_required def takeoff_pdf_page(): """Render one page of a project PDF as a PNG sized for fit-to-width preview (DVI-641).""" file_path = _takeoff_resolve_file(request.args.get("folder", ""), request.args.get("file", "")) if file_path is None or file_path.suffix.lower() != ".pdf": return "Invalid path", 400 try: page_no = int(request.args.get("page", "0")) except ValueError: return "Invalid page", 400 import io try: import fitz doc = fitz.open(str(file_path)) try: if page_no < 0 or page_no >= doc.page_count: return "Page out of range", 404 page = doc.load_page(page_no) zoom = min(2000.0 / max(page.rect.width, 1), 3.0) if zoom <= 0: zoom = 1.0 pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom)) png = pix.tobytes("png") finally: doc.close() except Exception as exc: # noqa: BLE001 — surface as error; UI falls back return f"Could not render page: {exc}", 500 return send_file(io.BytesIO(png), mimetype="image/png", download_name=f"page-{page_no + 1}.png") # Fields the per-field region OCR (DVI-645) may write a raw value into. Adds # vendor_project_code to _META_FIELDS: the user is pointing at an exact value, # so it is filled even though it is not an auto-discovered title-block field. _META_REGION_FIELDS = _META_FIELDS + ["vendor_project_code"] @app.route("/takeoff/extract-region", methods=["POST"]) @takeoff_workflow_required def takeoff_extract_region(): """OCR a user-drawn rectangle on one page of the previewed PDF (DVI-645). Body: {folder, file, page, rect:{x0,y0,x1,y1 as 0..1 fractions}, field?}. Without `field`, returns {fields:{…}} parsed from "LABEL: value" lines (the "OCR this region → best guess" flow). With `field`, returns {field, text} where text is the raw recognized text for that one targeted input.""" data = request.get_json(silent=True) or {} file_path = _takeoff_resolve_file(data.get("folder", ""), data.get("file", "")) if file_path is None: return jsonify(error="Invalid path"), 400 if file_path.suffix.lower() != ".pdf": return jsonify(error="OCR can only read PDF documents."), 400 try: page = int(data.get("page", 0)) except (TypeError, ValueError): return jsonify(error="Invalid page"), 400 rect = data.get("rect") or {} try: x0, y0 = float(rect.get("x0")), float(rect.get("y0")) x1, y1 = float(rect.get("x1")), float(rect.get("y1")) except (TypeError, ValueError): return jsonify(error="Invalid region"), 400 # Normalize to (min,max) and clamp to the page so a slightly out-of-bounds # drag (or a flipped selection) still maps to a valid clip. fx0, fx1 = sorted((max(0.0, min(1.0, x0)), max(0.0, min(1.0, x1)))) fy0, fy1 = sorted((max(0.0, min(1.0, y0)), max(0.0, min(1.0, y1)))) if (fx1 - fx0) < 0.005 or (fy1 - fy0) < 0.005: return jsonify(error="Selection too small — draw a larger box."), 400 field = (data.get("field") or "").strip() from ocr_engine import extract_region try: result = extract_region(file_path, page, (fx0, fy0, fx1, fy1), raw=bool(field)) except Exception as exc: # noqa: BLE001 — region OCR is best-effort prefill app.logger.warning("region OCR failed for %s: %s", file_path, exc) result = "" if field else {} if _is_history_capture_enabled("takeoff-bridge"): record_history( "takeoff-bridge", "region-ocr", actor=(session.get("user") or {}).get("name"), summary=f"Region OCR: {data.get('file', '')} p{page}", ref=data.get("file", ""), detail={"folder": data.get("folder", ""), "page": page, "field": field or None}, ) if field: if field not in _META_REGION_FIELDS: return jsonify(error="Unknown field"), 400 return jsonify(field=field, text=result or "") fields = {k: v for k, v in (result or {}).items() if k in _META_REGION_FIELDS and v} return jsonify(fields=fields) # --------------------------------------------------------------------------- # Routes — OCR Tool (DVI-490): upload PDF -> searchable PDF (background job) # --------------------------------------------------------------------------- def _ocr_job_dir(job_id: str) -> "Path | None": """Resolve a job folder under OCR_DIR, rejecting traversal.""" if not job_id or "/" in job_id or ".." in job_id: return None job_dir = (OCR_DIR / job_id).resolve() if not str(job_dir).startswith(str(OCR_DIR.resolve())): return None return job_dir def _ocr_write_status(job_dir: Path, status: dict) -> None: (job_dir / "status.json").write_text(json.dumps(status, indent=2)) def _ocr_read_status(job_dir: Path) -> dict: p = job_dir / "status.json" if not p.exists(): return {"status": "pending"} try: return json.loads(p.read_text()) except (json.JSONDecodeError, OSError): return {"status": "pending"} def _prune_ocr_jobs() -> None: """Remove OCR job folders older than OCR_JOB_TTL_DAYS.""" if not OCR_DIR.is_dir(): return cutoff = time.time() - OCR_JOB_TTL_DAYS * 86400 for child in OCR_DIR.iterdir(): try: if child.is_dir() and child.stat().st_mtime < cutoff: shutil.rmtree(str(child), ignore_errors=True) except OSError: continue def _run_ocr_job(job_dir: Path, in_path: Path, out_path: Path, mode: str, *, actor=None, orig_name=None) -> None: """Background thread: OCR the uploaded PDF into a searchable PDF.""" try: from ocr_engine import ocr_pdf, OcrEngineUnavailable try: result = ocr_pdf(in_path, out_path, mode=mode) except OcrEngineUnavailable as exc: _ocr_write_status(job_dir, {"status": "error", "error": str(exc), "filename": out_path.name}) return pages = len(result.pages) _ocr_write_status(job_dir, { "status": "done", "filename": out_path.name, "pages": pages, }) if _is_history_capture_enabled("ocr-tool"): record_history( "ocr-tool", "ocr-run", actor=actor, summary=f"OCR: {orig_name or out_path.name} ({pages}p)", ref=orig_name or "", detail={"pages": pages, "mode": mode}, ) except Exception as exc: # noqa: BLE001 — surface any OCR failure to the UI _ocr_write_status(job_dir, {"status": "error", "error": str(exc), "filename": out_path.name}) @app.route("/ocr/submit", methods=["POST"]) @login_required def ocr_submit(): """Accept a PDF upload and start a background OCR job.""" try: _prune_ocr_jobs() f = request.files.get("pdf") if not f or not f.filename: return jsonify(error="A PDF file is required"), 400 if not f.filename.lower().endswith(".pdf"): return jsonify(error="File must be a PDF"), 400 mode = request.form.get("mode", "skip-text") if mode not in ("skip-text", "redo-ocr", "force-ocr"): mode = "skip-text" try: OCR_DIR.mkdir(parents=True, exist_ok=True) except OSError as exc: return jsonify(error=f"OCR storage unavailable: {exc}"), 500 job_id = str(uuid.uuid4())[:8] job_dir = OCR_DIR / job_id job_dir.mkdir(parents=True, exist_ok=True) timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") base = secure_filename(f.filename) or "document.pdf" in_name = f"{timestamp}_{base}" in_path = job_dir / in_name f.save(in_path) out_name = f"{Path(base).stem}_searchable.pdf" out_path = job_dir / out_name _ocr_write_status(job_dir, {"status": "pending", "filename": out_name}) _ocr_actor = (session.get("user") or {}).get("name") t = threading.Thread( target=_run_ocr_job, args=(job_dir, in_path, out_path, mode), kwargs={"actor": _ocr_actor, "orig_name": f.filename}, daemon=True, ) t.start() return jsonify(job_id=job_id) except Exception as exc: # noqa: BLE001 — always return JSON to the browser return jsonify(error=f"Unexpected error: {exc}"), 500 @app.route("/ocr/status/") @login_required def ocr_status(job_id): job_dir = _ocr_job_dir(job_id) if not job_dir or not job_dir.is_dir(): return jsonify(status="error", error="Job not found"), 404 status = _ocr_read_status(job_dir) return jsonify(status=status.get("status", "pending"), filename=status.get("filename", ""), error=status.get("error", "")) @app.route("/ocr/download/") @login_required def ocr_download(job_id): job_dir = _ocr_job_dir(job_id) if not job_dir or not job_dir.is_dir(): abort(404) status = _ocr_read_status(job_dir) if status.get("status") != "done": abort(404) out_path = (job_dir / status.get("filename", "")).resolve() if not str(out_path).startswith(str(job_dir.resolve())) or not out_path.exists(): abort(404) return send_file(str(out_path), mimetype="application/pdf", as_attachment=True, download_name=out_path.name) @app.route("/ocr/backend") @login_required def ocr_backend_get(): """Return the selected OCR backend and the registered backend list.""" from ocr_engine import list_backends, load_backend return jsonify(backend=load_backend(), backends=list_backends()) @app.route("/ocr/backend", methods=["POST"]) @togen_admin_required def ocr_backend_post(): """Set the OCR backend. Admin only.""" from ocr_engine import OCR_BACKENDS, save_backend body = request.get_json(silent=True) or {} backend = (body.get("backend") or "").strip() if backend not in OCR_BACKENDS: return jsonify(error="Invalid OCR backend selected."), 400 save_backend(backend) return jsonify(ok=True, backend=backend) # --------------------------------------------------------------------------- # OCR Tool — Projects tab (DVI-646) # # OCR a file that already lives in the shared Projects tree (PDF, image, or # Office doc), present the searchable result inline, then optionally save the # searchable PDF back into the project directory. The browse/save endpoints are # gated to the Take-off Workflow group + Togen admins (same rule as the File # Browser's /fb/* routes via @takeoff_workflow_required). # --------------------------------------------------------------------------- def _run_ocr_project_job(job_dir: Path, in_path: Path, out_path: Path, mode: str, *, actor=None, orig_name=None) -> None: """Background thread: OCR an existing project file into a searchable PDF and ensure a text sidecar exists for in-document search.""" try: from ocr_engine import ocr_file, OcrEngineUnavailable try: result = ocr_file(in_path, out_path, mode=mode) except (OcrEngineUnavailable, ValueError) as exc: _ocr_write_status(job_dir, {"status": "error", "error": str(exc), "filename": out_path.name}) return # The local backend writes .txt; guarantee a sidecar regardless of # backend so /ocr/search has text to work with. sidecar = out_path.with_suffix(".txt") if not sidecar.exists(): try: sidecar.write_text(result.text or "", encoding="utf-8") except OSError: pass pages = len(result.pages) _ocr_write_status(job_dir, {"status": "done", "filename": out_path.name, "pages": pages}) if _is_history_capture_enabled("ocr-tool"): record_history( "ocr-tool", "ocr-project", actor=actor, summary=f"OCR (project): {orig_name or out_path.name} ({pages}p)", ref=orig_name or "", detail={"pages": pages, "mode": mode}, ) except Exception as exc: # noqa: BLE001 — surface any OCR failure to the UI _ocr_write_status(job_dir, {"status": "error", "error": str(exc), "filename": out_path.name}) @app.route("/ocr/process-project", methods=["POST"]) @takeoff_workflow_required def ocr_process_project(): """Start an OCR job on a file already in the shared Projects tree.""" try: from ocr_engine import is_processable _prune_ocr_jobs() body = request.get_json(silent=True) or {} rel = (body.get("path") or "").strip() src = _fb_safe_path(rel) if rel else None if src is None or not src.is_file(): return jsonify(error="File not found in the Projects directory."), 400 if not is_processable(src.name): return jsonify(error="This file type cannot be OCR-processed."), 400 mode = body.get("mode", "skip-text") if mode not in ("skip-text", "redo-ocr", "force-ocr"): mode = "skip-text" try: OCR_DIR.mkdir(parents=True, exist_ok=True) except OSError as exc: return jsonify(error=f"OCR storage unavailable: {exc}"), 500 job_id = str(uuid.uuid4())[:8] job_dir = OCR_DIR / job_id job_dir.mkdir(parents=True, exist_ok=True) in_name = secure_filename(src.name) or "document" in_path = job_dir / in_name shutil.copy2(str(src), str(in_path)) out_name = f"{Path(src.name).stem}_searchable.pdf" out_path = job_dir / out_name _ocr_write_status(job_dir, {"status": "pending", "filename": out_name}) _ocr_proj_actor = (session.get("user") or {}).get("name") threading.Thread(target=_run_ocr_project_job, args=(job_dir, in_path, out_path, mode), kwargs={"actor": _ocr_proj_actor, "orig_name": src.name}, daemon=True).start() src_dir = str(src.parent.relative_to(_FB_ROOT.resolve())) if src_dir == ".": src_dir = "" return jsonify(job_id=job_id, suggested_name=f"{Path(src.name).stem}-ocr_processed", source_dir=src_dir, source_name=src.name) except Exception as exc: # noqa: BLE001 — always return JSON to the browser return jsonify(error=f"Unexpected error: {exc}"), 500 @app.route("/ocr/view/") @login_required def ocr_view(job_id): """Serve a finished job's searchable PDF inline (for the preview iframe).""" job_dir = _ocr_job_dir(job_id) if not job_dir or not job_dir.is_dir(): abort(404) status = _ocr_read_status(job_dir) if status.get("status") != "done": abort(404) out_path = (job_dir / status.get("filename", "")).resolve() if not str(out_path).startswith(str(job_dir.resolve())) or not out_path.exists(): abort(404) return send_file(str(out_path), mimetype="application/pdf", as_attachment=False, download_name=out_path.name) @app.route("/ocr/search/") @login_required def ocr_search(job_id): """Search a finished job's OCR text. Returns match count + per-page hits so the UI can jump the inline viewer to the first matching page.""" job_dir = _ocr_job_dir(job_id) if not job_dir or not job_dir.is_dir(): return jsonify(error="Job not found"), 404 q = (request.args.get("q") or "").strip() if not q: return jsonify(count=0, matches=[], first_page=0) status = _ocr_read_status(job_dir) out_name = status.get("filename", "") sidecar = (job_dir / out_name).with_suffix(".txt") if out_name else None if not sidecar or not sidecar.exists(): return jsonify(count=0, matches=[], first_page=0) try: text = sidecar.read_text(encoding="utf-8", errors="replace") except OSError: return jsonify(count=0, matches=[], first_page=0) ql = q.lower() matches = [] total = 0 for i, ptext in enumerate(text.split("\f")): low = ptext.lower() n = low.count(ql) if n: total += n idx = low.find(ql) start = max(0, idx - 40) snippet = ptext[start:idx + len(q) + 40].replace("\n", " ").strip() matches.append({"page": i + 1, "count": n, "snippet": snippet}) return jsonify(count=total, matches=matches, first_page=(matches[0]["page"] if matches else 0)) @app.route("/ocr/save-project", methods=["POST"]) @takeoff_workflow_required def ocr_save_project(): """Save a finished OCR job's searchable PDF into the Projects tree. Returns {needs_confirm: true} (HTTP 200) when the target already exists and the caller has not set overwrite, so the UI can ask before clobbering.""" try: body = request.get_json(silent=True) or {} job_id = (body.get("job_id") or "").strip() dest_dir = (body.get("dir") or "").strip() filename = (body.get("filename") or "").strip() overwrite = bool(body.get("overwrite")) job_dir = _ocr_job_dir(job_id) if not job_dir or not job_dir.is_dir(): return jsonify(error="OCR job not found."), 404 status = _ocr_read_status(job_dir) if status.get("status") != "done": return jsonify(error="OCR is not finished yet."), 400 src_pdf = (job_dir / status.get("filename", "")).resolve() if not str(src_pdf).startswith(str(job_dir.resolve())) or not src_pdf.exists(): return jsonify(error="Processed file is unavailable."), 404 if not filename or "/" in filename or "\\" in filename or filename in (".", ".."): return jsonify(error="Invalid filename."), 400 if not filename.lower().endswith(".pdf"): filename += ".pdf" rel = f"{dest_dir}/{filename}" if dest_dir else filename dest = _fb_safe_path(rel) if dest is None: return jsonify(error="Invalid destination path."), 400 if dest.parent != _FB_ROOT.resolve() and not dest.parent.is_dir(): return jsonify(error="Destination directory does not exist."), 400 if dest.exists() and not overwrite: return jsonify(needs_confirm=True, filename=filename), 200 shutil.copy2(str(src_pdf), str(dest)) return jsonify(ok=True, path=str(dest.relative_to(_FB_ROOT.resolve())), filename=filename) except Exception as exc: # noqa: BLE001 — always return JSON to the browser return jsonify(error=f"Unexpected error: {exc}"), 500 # --------------------------------------------------------------------------- # Routes — single-item legacy workflow (unchanged) # --------------------------------------------------------------------------- @app.route("/upload", methods=["POST"]) @login_required def upload(): shop = request.files.get("shop") submittal = request.files.get("submittal") carrier = request.files.get("carrier") csv_file = request.files.get("csv") job_name = request.form.get("job_name", "").strip() if not (shop and shop.filename and submittal and submittal.filename and carrier and carrier.filename): return jsonify(error="shop, submittal, and carrier PDFs are required"), 400 for label, f in [("shop", shop), ("submittal", submittal), ("carrier", carrier)]: if not f.filename.lower().endswith(".pdf"): return jsonify(error=f"'{label}' must be a PDF file"), 400 if csv_file and csv_file.filename and not csv_file.filename.lower().endswith(".csv"): return jsonify(error="csv must be a .csv file"), 400 job_id = re.sub(r"[^\w\s\-\.]", "_", job_name)[:80].strip() if job_name else str(uuid.uuid4())[:8] job_folder = _safe_job_folder(job_id) if _find_generated(job_folder): return jsonify(job_id=job_id) job_folder.mkdir(parents=True, exist_ok=True) shop.save(job_folder / secure_filename(shop.filename)) submittal.save(job_folder / secure_filename(submittal.filename)) carrier.save(job_folder / secure_filename(carrier.filename)) if csv_file and csv_file.filename: csv_file.save(job_folder / secure_filename(csv_file.filename)) t = threading.Thread(target=_run_merge_single, args=(job_folder,), daemon=True) t.start() return jsonify(job_id=job_id) @app.route("/status/") @login_required def status(job_id): job_folder = _safe_job_folder(job_id) if not job_folder.is_dir(): abort(404) generated = _find_generated(job_folder) if generated: return jsonify(status="done", filename=generated.name) return jsonify(status="pending") @app.route("/preview/") @login_required def preview(job_id): job_folder = _safe_job_folder(job_id) generated = _find_generated(job_folder) if not generated: abort(404) return send_file(generated, mimetype="application/pdf") @app.route("/download/") @login_required def download_pdf(job_id): job_folder = _safe_job_folder(job_id) generated = _find_generated(job_folder) if not generated: abort(404) return send_file(generated, as_attachment=True, download_name=generated.name) @app.route("/single-carrier-viewer-url") @login_required def single_carrier_viewer_url(): """Return the Document Viewer URL for a generated carrier sheet.""" job_id = request.args.get("job_id", "") if not job_id: return jsonify({"error": "missing job_id"}), 400 job_folder = _safe_job_folder(job_id) generated = _find_generated(job_folder) if not generated: return jsonify({"error": "not found"}), 404 prefix = _load_shared_url_prefix() url = f"{prefix}/shared/{quote(job_folder.name)}/{quote(generated.name)}" # rel = path under SHARED_DIR, for the Print button's print-job file_ref (DVI-670) return jsonify({"url": url, "rel": f"{job_folder.name}/{generated.name}"}) # --------------------------------------------------------------------------- # Routes — Reprint Carrier Sheet (DVI-296) # --------------------------------------------------------------------------- @app.route("/reprint-list-projects", methods=["GET"]) @login_required def reprint_list_projects(): """Return project folder names under SHARED_DIR, sorted newest-to-oldest.""" folders = [] if SHARED_DIR.is_dir(): try: for entry in sorted( SHARED_DIR.iterdir(), key=lambda e: e.stat().st_mtime, reverse=True, ): if entry.is_dir() and not entry.name.startswith("."): folders.append(entry.name) except OSError: pass return jsonify({"folders": folders}) @app.route("/reprint-list-carriers", methods=["POST"]) @login_required def reprint_list_carriers(): """List carrier sheet PDFs in a project folder, ordered by CSV/JSON.""" data = request.get_json(silent=True) or {} folder_name = data.get("folder_name", "").strip() if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 folder_path = (SHARED_DIR / folder_name).resolve() if not str(folder_path).startswith(str(SHARED_DIR.resolve())) or not folder_path.is_dir(): return jsonify({"error": "Folder not found"}), 404 # Derive job name from metadata or folder name job_name = folder_name parts = folder_name.split(" - ") if len(parts) >= 3: job_name = parts[1].strip() elif len(parts) == 2: job_name = parts[1].strip() meta_path = folder_path / "project_metadata.json" if meta_path.is_file(): try: meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta.get("project_name"): job_name = meta["project_name"] except (json.JSONDecodeError, OSError): pass all_pdfs = list(folder_path.rglob("*.pdf")) carriers: list[dict] = [] seen: set[str] = set() # Strategy 1: Parse CSV for ordered items and find matching carrier files csv_path = None for f in folder_path.iterdir(): if f.suffix.lower() == ".csv" and f.is_file(): csv_path = f break if csv_path: try: carriers = _reprint_carriers_from_csv(csv_path, all_pdfs, folder_path, seen) except (OSError, csv.Error): pass # Strategy 2: state.json file mappings if not carriers: state_path = folder_path / "state.json" if state_path.is_file(): try: state = json.loads(state_path.read_text(encoding="utf-8")) for i, item_name in enumerate(state.get("items", [])): fm = state.get("files", {}).get(str(i), {}) carrier_name = fm.get("carrier") if not carrier_name: continue cp = folder_path / carrier_name if not cp.is_file(): continue rel = str(cp.relative_to(SHARED_DIR)) if rel in seen: continue seen.add(rel) shop_name = fm.get("shop") sub_name = fm.get("submittal") carriers.append({ "name": item_name, "carrier_path": rel, "shop_path": _reprint_rel(folder_path, shop_name, SHARED_DIR), "submittal_path": _reprint_rel(folder_path, sub_name, SHARED_DIR), }) except (json.JSONDecodeError, OSError): pass # Strategy 3: Scan folder for any carrier PDFs if not carriers: for pdf in sorted(all_pdfs, key=lambda f: f.name.lower()): stem_lower = pdf.stem.lower() rel = str(pdf.relative_to(SHARED_DIR)) if rel in seen: continue if stem_lower.endswith("carrierqr") or ( "carrier" in stem_lower and "generated" in stem_lower ): seen.add(rel) carriers.append({ "name": pdf.stem, "carrier_path": rel, "shop_path": None, "submittal_path": None, }) return jsonify({ "job_name": job_name, "folder_name": folder_name, "carriers": carriers, }) def _reprint_rel(folder: Path, filename: str | None, shared: Path) -> str | None: if not filename: return None p = folder / filename return str(p.relative_to(shared)) if p.is_file() else None def _reprint_find_pdf( url: str, all_pdfs: list[Path], shared_dir: Path, identifiers: list[str], suffix: str, ) -> str | None: """Locate a PDF by URL-extracted filename or by identifier+suffix matching.""" if url: filename = unquote(url.rstrip("/").rsplit("/", 1)[-1]) for pdf in all_pdfs: if pdf.name == filename: return str(pdf.relative_to(shared_dir)) idents = [s.strip().lower() for s in identifiers if s.strip()] if idents: for pdf in all_pdfs: if all(i in pdf.stem.lower() for i in idents) and pdf.stem.lower().endswith(suffix): return str(pdf.relative_to(shared_dir)) return None def _reprint_carriers_from_csv( csv_path: Path, all_pdfs: list[Path], folder_path: Path, seen: set[str], ) -> list[dict]: """Parse a project CSV and return carrier entries in row order.""" carriers: list[dict] = [] with open(csv_path, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) header = next(reader, []) header_lower = [h.strip().lower() for h in header] col = {} for i, h in enumerate(header_lower): if "tranid" in h: col.setdefault("tid", i) elif "structure" in h: col.setdefault("struct", i) elif h in ("itemline_item", "item"): col.setdefault("item", i) elif "part reference" in h: col.setdefault("pref", i) elif "qr label" in h: col.setdefault("curl", i) elif "drawingimage" in h: col.setdefault("surl", i) elif "submittal" in h and "link" in h: col.setdefault("burl", i) # Positional fallbacks for standard 12-column layout if "item" not in col and len(header) > 2: col.setdefault("item", 2) if "curl" not in col and len(header) > 11: col.setdefault("curl", 11) if "surl" not in col and len(header) > 9: col.setdefault("surl", 9) if "burl" not in col and len(header) > 10: col.setdefault("burl", 10) def _cell(row: list[str], key: str) -> str: idx = col.get(key) return row[idx].strip() if idx is not None and idx < len(row) else "" for row in reader: item = _cell(row, "item") struct = _cell(row, "struct") tid = _cell(row, "tid") pref = _cell(row, "pref") if not item and not struct: continue display = item or struct or pref or tid ids = [tid, pref, item] carrier_file = _reprint_find_pdf(_cell(row, "curl"), all_pdfs, SHARED_DIR, ids, "carrierqr") if not carrier_file or carrier_file in seen: continue seen.add(carrier_file) carriers.append({ "name": display, "carrier_path": carrier_file, "shop_path": _reprint_find_pdf(_cell(row, "surl"), all_pdfs, SHARED_DIR, ids, "shop"), "submittal_path": _reprint_find_pdf(_cell(row, "burl"), all_pdfs, SHARED_DIR, ids, "submittal"), }) return carriers # --------------------------------------------------------------------------- # Route — SCS save to project folder (DVI-309) # --------------------------------------------------------------------------- def _scs_find_job_pdf(job_folder: Path, sent_name: str, keyword: str, exclude: set) -> "Path | None": """Resolve a shop/submittal PDF in the job folder: client-sent filename first, keyword fallback.""" if sent_name: cand = job_folder / secure_filename(sent_name) if cand.is_file() and cand not in exclude: return cand for f in sorted(job_folder.iterdir()): if not f.is_file() or f.suffix.lower() != ".pdf" or f in exclude: continue name = f.name.lower() if "generated" in name: continue if keyword in name: return f return None @app.route("/scs-save-to-project", methods=["POST"]) @takeoff_workflow_required def scs_save_to_project(): """Copy an SCS carrier sheet (+ drawings) into a project folder and update its CSV records.""" data = request.get_json(silent=True) or {} folder_name = (data.get("folder_name") or "").strip() if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 project_folder = (SHARED_DIR / folder_name).resolve() if not str(project_folder).startswith(str(SHARED_DIR.resolve()) + os.sep): return jsonify({"error": "Invalid folder name"}), 400 if not project_folder.is_dir(): return jsonify({"error": "Project folder not found"}), 404 product_code = (data.get("product_code") or "").strip() if not product_code: return jsonify({"error": "Product Code is required"}), 400 append_row = bool(data.get("append_row", False)) _qr_base = _get_qr_base_url().rstrip("/") folder_url = f"{_qr_base}/{quote(folder_name, safe='')}" # shop/submittal URLs: None = leave the CSV cell unchanged shop_url: "str | None" = None submittal_url: "str | None" = None carrier_rel = (data.get("carrier_path") or "").strip() if carrier_rel: # Reprint mode: source carrier already lives in SHARED_DIR; copy only across folders. if ".." in carrier_rel: return jsonify({"error": "Invalid carrier path"}), 400 carrier_src = (SHARED_DIR / carrier_rel).resolve() if not str(carrier_src).startswith(str(SHARED_DIR.resolve()) + os.sep) or not carrier_src.is_file(): return jsonify({"error": "Carrier sheet not found"}), 404 copy_sources = [] if carrier_src.parent == project_folder else [carrier_src] # J/K point at the source project's existing files (not copied) when known shop_rel = (data.get("shop_path") or "").strip() submittal_rel = (data.get("submittal_path") or "").strip() if shop_rel and ".." not in shop_rel: shop_url = f"{_qr_base}/{quote(shop_rel, safe='/')}" if submittal_rel and ".." not in submittal_rel: submittal_url = f"{_qr_base}/{quote(submittal_rel, safe='/')}" else: job_id = (data.get("job_id") or "").strip() if not job_id: return jsonify({"error": "Missing job_id"}), 400 job_folder = _safe_job_folder(job_id) carrier_src = _find_generated(job_folder) if not carrier_src: return jsonify({"error": "No generated carrier sheet found"}), 404 shop_src = _scs_find_job_pdf(job_folder, data.get("shop_filename") or "", "shop", {carrier_src}) submittal_src = _scs_find_job_pdf( job_folder, data.get("submittal_filename") or "", "submittal", {carrier_src} | ({shop_src} if shop_src else set()), ) copy_sources = [carrier_src] + [p for p in (shop_src, submittal_src) if p] if shop_src: shop_url = f"{folder_url}/{quote(shop_src.name, safe='')}" if submittal_src: submittal_url = f"{folder_url}/{quote(submittal_src.name, safe='')}" carrier_url = f"{folder_url}/{quote(carrier_src.name, safe='')}" # Copy files (overwrite on collision — same as the publish flow; idempotent) saved_files = [] for src in copy_sources: shutil.copy2(str(src), str(project_folder / src.name)) saved_files.append(src.name) # Locate the project CSV: state.json csv_filename first, else a single *.csv csv_path = None csv_warning = None state_path = project_folder / "state.json" if state_path.is_file(): try: state = json.loads(state_path.read_text(encoding="utf-8")) name = state.get("csv_filename") if name and (project_folder / name).is_file(): csv_path = project_folder / name except (json.JSONDecodeError, OSError): pass if csv_path is None: csvs = sorted(f for f in project_folder.iterdir() if f.is_file() and f.suffix.lower() == ".csv") if len(csvs) == 1: csv_path = csvs[0] elif len(csvs) > 1: csv_warning = "Multiple CSV files found — project records not updated" else: csv_warning = "No CSV found — project records not updated" if csv_warning: return jsonify({"ok": True, "saved_files": saved_files, "csv_warning": csv_warning}) # Update the CSV, mirroring the Take-off Bridge process: match column C (index 2), # pad rows to 12 columns, write J(9)/K(10)/L(11) URLs. with open(csv_path, newline="", encoding="utf-8-sig") as fh: rows = list(csv.reader(fh)) code_lower = product_code.lower() matched = [ idx for idx, row in enumerate(rows) if idx > 0 and len(row) >= 3 and row[2].strip().lower() == code_lower ] appended = False if matched: # Also covers the append_row retry racing a concurrent edit: update, don't duplicate. for idx in matched: row = rows[idx] while len(row) < 12: row.append("") if shop_url is not None: row[9] = shop_url if submittal_url is not None: row[10] = submittal_url row[11] = carrier_url elif not append_row: return jsonify({ "ok": True, "saved_files": saved_files, "csv_name": csv_path.name, "csv_match": False, }) else: new_row = [""] * 12 new_row[1] = (data.get("structure_id") or "").strip() new_row[2] = product_code new_row[5] = (data.get("weight") or "").strip() new_row[6] = (data.get("structure_size") or "").strip() new_row[7] = (data.get("part_ref") or "").strip() if shop_url is not None: new_row[9] = shop_url if submittal_url is not None: new_row[10] = submittal_url new_row[11] = carrier_url rows.append(new_row) appended = True with open(csv_path, "w", newline="", encoding="utf-8") as fh: csv.writer(fh).writerows(rows) return jsonify({ "ok": True, "saved_files": saved_files, "csv_name": csv_path.name, "csv_match": True, "appended": appended, "updated_rows": len(matched), "product_code": product_code, }) # --------------------------------------------------------------------------- # Routes — Sync Admin (DVI-235) # --------------------------------------------------------------------------- RCLONE_LOG_PATH = Path("/home/manager/rclone.log") RCLONE_RESYNC_SCRIPT = "/home/manager/rclone_resync.sh" RCLONE_FORCE_RESYNC_SCRIPT = "/home/manager/rclone_force_resync.sh" SYNC_ADMIN_DEFAULT_LINES = 40 @app.route("/sync-admin/log") @takeoff_workflow_required def sync_admin_log(): if not RCLONE_LOG_PATH.is_file(): return jsonify(error="Log file not found at " + str(RCLONE_LOG_PATH)), 404 try: lines_requested = request.args.get("lines", SYNC_ADMIN_DEFAULT_LINES, type=int) lines_requested = max(1, min(lines_requested, 500)) with open(RCLONE_LOG_PATH, "rb") as f: # Read from end of file efficiently f.seek(0, 2) size = f.tell() # Read up to 64KB from end — enough for ~500 lines chunk_size = min(size, 65536) f.seek(max(0, size - chunk_size)) raw = f.read() text = raw.decode("utf-8", errors="replace") all_lines = text.splitlines() tail = all_lines[-lines_requested:] needs_resync = False needs_lock_fix = False has_errors = False if tail: last_line = tail[-1] if "Bisync aborted" in last_line and "Must run --resync to recover" in last_line: needs_resync = True for line in tail: if "failed to bisync: prior lock file found:" in line: needs_lock_fix = True if re.search(r'\b(?:error|failed|aborted)\b', line, re.IGNORECASE): has_errors = True return jsonify( lines=tail, needs_resync=needs_resync, needs_lock_fix=needs_lock_fix, has_errors=has_errors, total_lines=len(all_lines), ) except Exception as exc: return jsonify(error=str(exc)), 500 @app.route("/sync-admin/resync", methods=["POST"]) @takeoff_workflow_required def sync_admin_resync(): if not os.path.isfile(RCLONE_RESYNC_SCRIPT): return jsonify(error="Resync script not found at " + RCLONE_RESYNC_SCRIPT), 404 try: subprocess.Popen( ["/bin/bash", RCLONE_RESYNC_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) record_history( "sync-admin", "resync", actor=(session.get("user") or {}).get("name"), summary="Resync initiated", ) return jsonify(ok=True, message="Resync script launched.") except Exception as exc: return jsonify(error=str(exc)), 500 @app.route("/sync-admin/force-resync", methods=["POST"]) @takeoff_workflow_required def sync_admin_force_resync(): """Stop running rclone, remove the lock file, and run a fresh sync.""" if not os.path.isfile(RCLONE_FORCE_RESYNC_SCRIPT): return jsonify(error="Force-resync script not found at " + RCLONE_FORCE_RESYNC_SCRIPT), 404 try: subprocess.Popen( ["/bin/bash", RCLONE_FORCE_RESYNC_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) record_history( "sync-admin", "force-resync", actor=(session.get("user") or {}).get("name"), summary="Force resync initiated (lock cleared)", ) return jsonify(ok=True, message="Force-resync script launched. rclone will be stopped, lock cleared, and a fresh sync started.") except Exception as exc: return jsonify(error=str(exc)), 500 @app.route("/sync-admin/log-tail") @takeoff_workflow_required def sync_admin_log_tail(): """Return the last 50 lines of the rclone log for support request pre-population.""" if not RCLONE_LOG_PATH.is_file(): return jsonify(error="Log file not found.", text=""), 404 try: with open(RCLONE_LOG_PATH, "rb") as f: f.seek(0, 2) size = f.tell() chunk_size = min(size, 65536) f.seek(max(0, size - chunk_size)) raw = f.read() text = raw.decode("utf-8", errors="replace") lines = text.splitlines()[-50:] return jsonify(text="\n".join(lines)) except Exception as exc: return jsonify(error=str(exc), text=""), 500 # --------------------------------------------------------------------------- # Togen Admin — version info & update trigger # --------------------------------------------------------------------------- _GITHUB_REPO = "andycrawford/Togen" _TOGEN_APP_DIR = Path(__file__).resolve().parent # DVI-768: the version pane (and every admin's Home-tile poll, every 5 min) hits # the GitHub API, which unauthenticated rate-limits to 60 req/hr/IP — several # admins easily exceed that and surface a "GitHub API returned 429" error. A # shared server-side cache collapses all callers onto one upstream request per # TTL window, and on any upstream error we serve the last good result instead of # bubbling up 429/timeout to the UI. _GH_VERSION_CACHE = {"data": None, "ts": 0.0} _GH_VERSION_TTL = 300 # seconds _GH_VERSION_LOCK = threading.Lock() def _fetch_github_latest(force=False): """Return (remote_info_dict, error_str) for the latest main commit. Caches successful results for ``_GH_VERSION_TTL`` and, on any upstream error (including 429 rate limiting or timeouts), serves the last cached result rather than failing — so the version pane stays usable. Only when no cached result exists is an error returned. """ import time as _t now = _t.time() with _GH_VERSION_LOCK: cached = _GH_VERSION_CACHE.get("data") cached_ts = _GH_VERSION_CACHE.get("ts", 0.0) if cached and not force and (now - cached_ts) < _GH_VERSION_TTL: return cached, None try: gh_headers = {"Accept": "application/vnd.github.v3+json"} gh_token = os.environ.get("GITHUB_TOKEN") if gh_token: gh_headers["Authorization"] = f"Bearer {gh_token}" gh_resp = _http_requests.get( f"https://api.github.com/repos/{_GITHUB_REPO}/commits/main", headers=gh_headers, timeout=15, ) if gh_resp.status_code != 200: if cached: return cached, None return None, f"GitHub API returned {gh_resp.status_code}" gh_data = gh_resp.json() remote_info = { "hash": gh_data["sha"][:12], "date": gh_data["commit"]["author"]["date"], "message": gh_data["commit"]["message"].split("\n")[0], } with _GH_VERSION_LOCK: _GH_VERSION_CACHE["data"] = remote_info _GH_VERSION_CACHE["ts"] = now return remote_info, None except Exception as exc: if cached: return cached, None return None, f"GitHub version check failed: {exc}" @app.route("/togen-admin/version") @togen_admin_required def togen_admin_version(): """Return deployed commit info and latest GitHub commit for comparison.""" version_file = _TOGEN_APP_DIR / "VERSION" try: if not version_file.is_file(): return jsonify(error="VERSION file not found. Redeploy required."), 500 ver = json.loads(version_file.read_text()) local_info = { "hash": ver["hash"][:12], "date": ver["date"], "message": ver.get("message", ""), } except Exception as exc: return jsonify(error=f"Local version check failed: {exc}"), 500 remote_info, gh_err = _fetch_github_latest() if gh_err: return jsonify(error=gh_err), 502 update_available = local_info["hash"] != remote_info["hash"] return jsonify( local=local_info, remote=remote_info, update_available=update_available, ) @app.route("/togen-admin/update", methods=["POST"]) @togen_admin_required def togen_admin_update(): """Trigger a git pull and service restart (wired by Linux Automation Engineer).""" update_script = Path("/opt/togen/update.sh") if not update_script.is_file(): return jsonify(error="Update script not yet configured. Contact your system administrator."), 501 try: subprocess.Popen( ["/bin/bash", str(update_script)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) record_history( "togen-admin", "update-applied", actor=(session.get("user") or {}).get("name"), summary="Immediate update applied (git pull + service restart)", ) return jsonify(ok=True, message="Update started. The application will restart momentarily.") except Exception as exc: return jsonify(error=str(exc)), 500 # --------------------------------------------------------------------------- # Togen Admin — scheduled update (DVI-768) # --------------------------------------------------------------------------- UPDATE_SCHEDULE_FILE = _TOGEN_APP_DIR / "update_schedule.json" _SCHEDULED_UPDATE_NOTIFY_EMAIL = "help@voyagetech.com" _UPDATE_SCHEDULE_LOCK = threading.Lock() def _load_update_schedule(): """Return the persisted scheduled-update record, or a disabled default.""" try: if UPDATE_SCHEDULE_FILE.is_file(): return json.loads(UPDATE_SCHEDULE_FILE.read_text()) except Exception: app.logger.exception("Failed to read update schedule") return {"enabled": False, "version": "", "scheduled_for": "", "status": "idle"} def _save_update_schedule(sched): try: UPDATE_SCHEDULE_FILE.write_text(json.dumps(sched, indent=2)) except OSError: app.logger.exception("Failed to write update schedule") def _next_2am_central_utc(): """Return the next 02:00 America/Chicago instant as a naive UTC datetime.""" try: from zoneinfo import ZoneInfo central = ZoneInfo("America/Chicago") now_c = _datetime_type.now(central) target = now_c.replace(hour=2, minute=0, second=0, microsecond=0) if target <= now_c: target += _timedelta_type(days=1) return target.astimezone(_dt_timezone.utc).replace(tzinfo=None) except Exception: # Fallback: 2am next day in server-local time, treated as UTC. target = _datetime_type.utcnow().replace(hour=2, minute=0, second=0, microsecond=0) if target <= _datetime_type.utcnow(): target += _timedelta_type(days=1) return target @app.route("/togen-admin/scheduled-update") @togen_admin_required def togen_admin_scheduled_update_get(): """Return the current scheduled-update state (DVI-768).""" return jsonify(schedule=_load_update_schedule()) @app.route("/togen-admin/scheduled-update", methods=["POST"]) @togen_admin_required def togen_admin_scheduled_update_post(): """Enable or cancel a scheduled update for 2:00 AM next morning (DVI-768). Body: {enabled: bool, version: str}. When enabled, ``version`` is the public version string to display to users once the update completes; the update is applied by the background scheduler at the next 2:00 AM Central. """ body = request.get_json(silent=True) or {} enabled = bool(body.get("enabled")) if not enabled: _save_update_schedule({"enabled": False, "version": "", "scheduled_for": "", "status": "idle"}) return jsonify(ok=True, schedule=_load_update_schedule()) version = str(body.get("version", "")).strip() if not version: return jsonify(error="Version is required when scheduling an update."), 400 scheduled_for = _next_2am_central_utc().strftime("%Y-%m-%dT%H:%M:%SZ") user = session.get("user", {}) sched = { "enabled": True, "version": version, "scheduled_for": scheduled_for, "status": "pending", "created_by": (user.get("email") or user.get("name") or ""), "created_at": _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"), } _save_update_schedule(sched) record_history( "togen-admin", "update-scheduled", actor=(session.get("user") or {}).get("name"), summary=f"Update v{version} scheduled for {scheduled_for}", ) return jsonify(ok=True, schedule=sched) def _send_scheduled_update_email(sched, remote, gh_err): """Email help@voyagetech.com that a scheduled update was applied (DVI-768).""" version = sched.get("version", "") scheduled_for = sched.get("scheduled_for", "") applied_at = sched.get("applied_at", "") created_by = sched.get("created_by", "") if remote: gh_block = ( f"
  • Commit: {remote.get('hash', '')}
    • " f"
  • Committed: {remote.get('date', '')}
    • " f"
  • Message: {remote.get('message', '')}
    • " ) else: gh_block = f"
  • GitHub commit details unavailable: {gh_err or 'unknown error'}
    • " html = ( "

    Togen Scheduled Update Applied

    " "

    A scheduled application update has been applied per schedule.

    " "
      " f"
    • Scheduled for: {scheduled_for}
      • " f"
    • Applied at (UTC): {applied_at}
      • " f"
    • Scheduled by: {created_by}
      • " f"
    • Version shown to users: {version}
      • " "
    " "

    GitHub update information

    " f"
      {gh_block}
    " ) try: notifications.send_html_email( _SCHEDULED_UPDATE_NOTIFY_EMAIL, "Togen scheduled update applied", html, ) except Exception: app.logger.exception("Failed to send scheduled-update email") def _maybe_run_scheduled_update(): """If a scheduled update is due, set the public version and apply it.""" with _UPDATE_SCHEDULE_LOCK: sched = _load_update_schedule() if not sched.get("enabled") or sched.get("status") != "pending": return scheduled_for = sched.get("scheduled_for") if not scheduled_for: return now_str = _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") if now_str < scheduled_for: return update_script = Path("/opt/togen/update.sh") if not update_script.is_file(): app.logger.warning("Scheduled update due but update.sh missing; leaving pending") return version = sched.get("version", "").strip() remote, gh_err = _fetch_github_latest(force=True) if version: try: PUBLIC_VERSION_FILE.write_text(version + "\n") except OSError: app.logger.exception("Scheduled update: failed to write public version") # Mark applied and disable BEFORE launching the update — update.sh # restarts the service, which restarts this loop; persisting the # applied status first prevents a re-run after restart. sched["status"] = "applied" sched["enabled"] = False sched["applied_at"] = now_str _save_update_schedule(sched) _send_scheduled_update_email(sched, remote, gh_err) try: subprocess.Popen( ["/bin/bash", str(update_script)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) app.logger.info("Scheduled update launched (version=%s)", version) except Exception: app.logger.exception("Scheduled update: failed to launch update.sh") # --------------------------------------------------------------------------- # Togen Admin — tool visibility (DVI-254) # --------------------------------------------------------------------------- @app.route("/togen-admin/tool-visibility") @togen_admin_required def togen_admin_tool_visibility_get(): """Return the current enabled/disabled state for each manageable tool.""" return jsonify(tools=_load_tool_visibility()) @app.route("/togen-admin/tool-visibility", methods=["POST"]) @togen_admin_required def togen_admin_tool_visibility_post(): """Save tool visibility settings. Body: {tools: {tool_id: bool, ...}}.""" body = request.get_json(silent=True) or {} incoming = body.get("tools", {}) visibility = _load_tool_visibility() for tid in _ALL_TOOL_IDS: if tid in incoming: visibility[tid] = bool(incoming[tid]) try: TOOL_VISIBILITY_FILE.write_text(json.dumps(visibility, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify(ok=True, tools=visibility) @app.route("/togen-admin/toolbar-visibility") @togen_admin_required def togen_admin_toolbar_visibility_get(): """Return Admins/Users enable state for each top-bar function (DVI-616).""" return jsonify(toolbars=_load_toolbar_visibility()) @app.route("/togen-admin/toolbar-visibility", methods=["POST"]) @togen_admin_required def togen_admin_toolbar_visibility_post(): """Save toolbar visibility. Body: {toolbars: {id: {admins: bool, users: bool}}}.""" body = request.get_json(silent=True) or {} incoming = body.get("toolbars", {}) visibility = _load_toolbar_visibility() for tid in _TOOLBAR_IDS: row = incoming.get(tid) if isinstance(row, dict): visibility[tid] = { "admins": bool(row.get("admins", visibility[tid]["admins"])), "users": bool(row.get("users", visibility[tid]["users"])), } try: TOOLBAR_VISIBILITY_FILE.write_text(json.dumps(visibility, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify(ok=True, toolbars=visibility) @app.route("/togen-admin/tool-order") @togen_admin_required def togen_admin_tool_order_get(): """Return the current tool display order.""" return jsonify(order=_load_tool_order()) @app.route("/togen-admin/tool-order", methods=["POST"]) @togen_admin_required def togen_admin_tool_order_post(): """Save tool display order. Body: {order: [tool_id, ...]}.""" body = request.get_json(silent=True) or {} incoming = body.get("order", []) if not isinstance(incoming, list): return jsonify(error="order must be a list"), 400 seen, valid = set(), [] for tid in incoming: if tid in _ALL_TOOL_IDS and tid not in seen: seen.add(tid) valid.append(tid) for tid in _ALL_TOOL_IDS: if tid not in seen: seen.add(tid) valid.append(tid) try: TOOL_ORDER_FILE.write_text(json.dumps(valid, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save order: {exc}"), 500 return jsonify(ok=True, order=valid) @app.route("/togen-admin/tool-overrides") @togen_admin_required def togen_admin_tool_overrides_get(): """Return per-tool display name/description overrides (DVI-588).""" return jsonify(overrides=_load_tool_overrides()) @app.route("/togen-admin/tool-overrides", methods=["POST"]) @togen_admin_required def togen_admin_tool_overrides_post(): """Save one tool's name/description override (DVI-588). Body: {tool_id, name, description}. Blank name or description clears that field so the tool falls back to its built-in default; clearing both removes the tool's entry entirely. """ body = request.get_json(silent=True) or {} tid = str(body.get("tool_id", "")).strip() if tid not in _ALL_TOOL_IDS: return jsonify(error="Unknown tool"), 400 name = str(body.get("name", "")).strip() desc = str(body.get("description", "")).strip() overrides = _load_tool_overrides() entry = {} if name: entry["name"] = name if desc: entry["description"] = desc if entry: overrides[tid] = entry else: overrides.pop(tid, None) try: TOOL_OVERRIDES_FILE.write_text(json.dumps(overrides, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save overrides: {exc}"), 500 return jsonify(ok=True, overrides=overrides) # --------------------------------------------------------------------------- # Togen Admin — Work Order type visibility/order (DVI-349) # --------------------------------------------------------------------------- @app.route("/togen-admin/wor-type-visibility") @togen_admin_required def togen_admin_wor_type_visibility_get(): """Return the enabled/disabled state for each Work Order type.""" return jsonify(types=_load_wor_type_visibility()) @app.route("/togen-admin/wor-type-visibility", methods=["POST"]) @togen_admin_required def togen_admin_wor_type_visibility_post(): """Save Work Order type visibility. Body: {types: {wor_type: bool, ...}}.""" body = request.get_json(silent=True) or {} incoming = body.get("types", {}) visibility = _load_wor_type_visibility() for tid in _WOR_TYPE_IDS: if tid in incoming: visibility[tid] = bool(incoming[tid]) try: WOR_TYPE_VISIBILITY_FILE.write_text(json.dumps(visibility, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify(ok=True, types=visibility) @app.route("/togen-admin/wor-type-order") @togen_admin_required def togen_admin_wor_type_order_get(): """Return the current Work Order type display order.""" return jsonify(order=_load_wor_type_order()) @app.route("/togen-admin/wor-type-order", methods=["POST"]) @togen_admin_required def togen_admin_wor_type_order_post(): """Save Work Order type display order. Body: {order: [wor_type, ...]}.""" body = request.get_json(silent=True) or {} incoming = body.get("order", []) if not isinstance(incoming, list): return jsonify(error="order must be a list"), 400 seen, valid = set(), [] for tid in incoming: if tid in _WOR_TYPE_IDS and tid not in seen: seen.add(tid) valid.append(tid) for tid in _WOR_TYPE_IDS: if tid not in seen: seen.add(tid) valid.append(tid) try: WOR_TYPE_ORDER_FILE.write_text(json.dumps(valid, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save order: {exc}"), 500 return jsonify(ok=True, order=valid) # --------------------------------------------------------------------------- # Routes — User Settings (DVI-274) # --------------------------------------------------------------------------- @app.route("/user-settings") @login_required def user_settings_get(): """Return the current user's personal settings.""" email = session["user"].get("email", "") return jsonify(settings=_user_settings_with_grid(email)) @app.route("/user-settings", methods=["POST"]) @login_required def user_settings_post(): """Save the current user's personal settings.""" email = session["user"].get("email", "") body = request.get_json(silent=True) or {} settings = _load_user_settings(email) if "tool_visibility" in body: tv = body["tool_visibility"] if isinstance(tv, dict): if tv: # togen-admin is per-user togglable (DVI-524) even though it is not # an admin-managed tool, so it is accepted alongside _ALL_TOOL_IDS. allowed_vis = list(_ALL_TOOL_IDS) + ["togen-admin"] settings["tool_visibility"] = { tid: bool(tv[tid]) for tid in allowed_vis if tid in tv } else: settings.pop("tool_visibility", None) if "tool_order" in body: incoming = body["tool_order"] if isinstance(incoming, list): if incoming: # Beyond the admin-managed tools, the per-user order may carry the # Togen Admin tile and "separator" markers (DVI-524). Dedup keeps a # single separator at its submitted position. allowed_order = set(_ALL_TOOL_IDS) | {"togen-admin", "separator"} seen, valid = set(), [] for tid in incoming: if tid in allowed_order and tid not in seen: seen.add(tid) valid.append(tid) for tid in _ALL_TOOL_IDS: if tid not in seen: seen.add(tid) valid.append(tid) settings["tool_order"] = valid else: settings.pop("tool_order", None) if "theme" in body: if body["theme"] in ("dark", "light", "system"): if body["theme"] == "system": settings.pop("theme", None) else: settings["theme"] = body["theme"] if "notifications" in body: grid = body["notifications"] if isinstance(grid, dict): wo = _notification_grid_to_storage(grid) if wo: settings["wo_notifications"] = wo else: settings.pop("wo_notifications", None) # Per-user SCADA notification prefs (DVI-465). Mirrored between the SCADA # View Settings tab and the User Settings Notifications pane. if "scada_notifications" in body: sn = body["scada_notifications"] if isinstance(sn, dict): settings["scada_notifications"] = { "stale_data": bool(sn.get("stale_data")), "out_of_range": bool(sn.get("out_of_range")), } # Background-mode override (DVI-364). A blank/null value clears the override # so the user inherits the company default; a valid mode overrides it. if "bg_mode_override" in body: val = body["bg_mode_override"] if val in (None, ""): settings.pop("bg_mode_override", None) elif val in USER_BG_MODES: settings["bg_mode_override"] = val else: return jsonify(error="Invalid background mode."), 400 # Per-user pane opacity overrides (DVI-366). Separate light/dark keys; a # blank/null value clears that key (inherit the company default). for key in ("bg_opacity_light", "bg_opacity_dark"): if key in body: val = body[key] if val in (None, ""): settings.pop(key, None) else: opacity = _coerce_bg_opacity(val) if opacity is None: return jsonify(error="Opacity must be an integer between 40 and 100."), 400 settings[key] = opacity # Per-user home tile grid overrides (DVI-512). "tile_columns": a blank/null # value clears the override (inherit the company default); otherwise 3-9 or # "unlimited". "tile_view": grid or list presentation, persisted across reloads. if "tile_columns" in body: val = body["tile_columns"] if val in (None, "", "default"): settings.pop("tile_columns", None) else: tile_columns = _coerce_tile_columns(val) if tile_columns is None: return jsonify(error='Tile columns must be 3-9 or "unlimited".'), 400 settings["tile_columns"] = tile_columns if "tile_view" in body: val = body["tile_view"] if val in TILE_VIEW_MODES: settings["tile_view"] = val else: return jsonify(error="Invalid tile view."), 400 # Per-user side padding override (DVI-515). Blank/null clears the override # so the home grid inherits the company default; otherwise 0-40 (%). if "tile_side_padding" in body: val = body["tile_side_padding"] if val in (None, "", "default"): settings.pop("tile_side_padding", None) else: tile_side_padding = _coerce_tile_side_padding(val) if tile_side_padding is None: return jsonify(error="Side padding must be an integer between 0 and 40."), 400 settings["tile_side_padding"] = tile_side_padding try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 resp = dict(settings) resp["notifications"] = _notification_grid_from_settings(settings) return jsonify(ok=True, settings=resp) # Per-user custom background (DVI-518). Upload/serve/remove a personal image used # as the Togen interface background when the user's bg override is "custom". @app.route("/user-settings/custom-bg") @login_required def user_settings_custom_bg_get(): """Serve the current user's uploaded custom background image; 404 if absent.""" email = session["user"].get("email", "") fp = _find_custom_bg(email) if not fp: abort(404) mime = CUSTOM_BG_EXT_MIMES.get(fp.suffix.lstrip(".").lower(), "application/octet-stream") return send_file(fp, mimetype=mime) @app.route("/user-settings/custom-bg", methods=["POST"]) @login_required def user_settings_custom_bg_post(): """Accept a per-user custom background image upload (DVI-518). Validates type and size, replaces any previously uploaded image, records the extension in user settings, and switches the user's background override to "custom" so the new image takes effect immediately. """ email = session["user"].get("email", "") f = request.files.get("background") if f is None or not f.filename: return jsonify(error="No file provided."), 400 ext = f.filename.rsplit(".", 1)[-1].lower() if "." in f.filename else "" if ext not in CUSTOM_BG_EXT_MIMES: return jsonify(error="Unsupported image type. Use PNG, JPG, WEBP, GIF, or SVG."), 400 f.stream.seek(0, os.SEEK_END) size = f.stream.tell() f.stream.seek(0) if size > CUSTOM_BG_MAX_BYTES: return jsonify(error="Image too large (max 8 MB)."), 400 CUSTOM_BG_DIR.mkdir(parents=True, exist_ok=True) # Remove any previously stored custom bg with a different extension so a user # never has two files on disk. old = _find_custom_bg(email) if old and old.suffix.lstrip(".").lower() != ext: try: old.unlink() except OSError: pass f.save(str(_custom_bg_path(email, ext))) settings = _load_user_settings(email) settings["custom_bg_ext"] = ext settings["bg_mode_override"] = "custom" try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 resp = dict(settings) resp["notifications"] = _notification_grid_from_settings(settings) return jsonify(ok=True, settings=resp) @app.route("/user-settings/custom-bg", methods=["DELETE"]) @login_required def user_settings_custom_bg_delete(): """Remove the current user's custom background and clear the "custom" override.""" email = session["user"].get("email", "") fp = _find_custom_bg(email) if fp: try: fp.unlink() except OSError: pass settings = _load_user_settings(email) settings.pop("custom_bg_ext", None) if settings.get("bg_mode_override") == "custom": settings.pop("bg_mode_override", None) try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 resp = dict(settings) resp["notifications"] = _notification_grid_from_settings(settings) return jsonify(ok=True, settings=resp) @app.route("/notifications-settings", methods=["POST"]) @login_required def notifications_settings_post(): """Persist the current user's notification grid (DVI-335). Stores the friendly 3x5 grid as the core module's event-keyed ``wo_notifications`` block inside the per-user settings JSON. """ # Defense in depth: the WO notifications grid is only available to Work # Order Admins (the UI hides it for everyone else), so non-admins must not # be able to persist WO notification prefs via a direct POST. if not session["user"].get("wor_admin"): return jsonify(error="Access denied: Work Order Admin group membership required."), 403 email = session["user"].get("email", "") body = request.get_json(silent=True) or {} grid = body.get("notifications") if not isinstance(grid, dict): return jsonify(error="Missing 'notifications' grid"), 400 settings = _load_user_settings(email) wo = _notification_grid_to_storage(grid) if wo: settings["wo_notifications"] = wo # Persist the email so the recipient scan in notifications.py can resolve # this admin even if the sanitized filename ever differs from the address. if email: settings["email"] = email else: settings.pop("wo_notifications", None) try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify(ok=True, notifications=_notification_grid_from_settings(settings)) @app.route("/wor-primary", methods=["GET"]) @wor_admin_required def wor_primary_get(): """Return the Primary recipient lists keyed by WO type (DVI-405).""" return jsonify(primary=_load_wor_primary()) @app.route("/wor-primary", methods=["POST"]) @wor_admin_required def wor_primary_post(): """Add or remove a Primary recipient email for a WO type (DVI-405).""" body = request.get_json(silent=True) or {} wor_type = body.get("wor_type", "") action = body.get("action", "") email = (body.get("email") or "").strip() if wor_type not in _NOTIFY_GRID_TYPES: return jsonify(error="Invalid work order type."), 400 if action not in ("add", "remove"): return jsonify(error="Invalid action."), 400 norm = email.lower() if action == "add" and ("@" not in norm or "." not in norm.split("@")[-1]): return jsonify(error="A valid email address is required."), 400 primary = _load_wor_primary() emails = primary.get(wor_type, []) if action == "add": if norm not in emails: emails.append(norm) else: emails = [e for e in emails if e != norm] primary[wor_type] = emails try: _save_wor_primary(primary) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True, primary=_load_wor_primary()) @app.route("/wor-policy", methods=["GET"]) @wor_admin_required def wor_policy_get(): """Return the Policy-enforced notifications grid (DVI-597).""" return jsonify(policy=_load_wor_policy_grid()) @app.route("/wor-policy", methods=["POST"]) @wor_admin_required def wor_policy_post(): """Persist the Policy-enforced notifications grid (DVI-597).""" body = request.get_json(silent=True) or {} grid = body.get("policy") if not isinstance(grid, dict): return jsonify(error="Invalid policy grid."), 400 try: _save_wor_policy_grid(grid) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True, policy=_load_wor_policy_grid()) # --------------------------------------------------------------------------- # Routes — Support Request (DVI-246) # --------------------------------------------------------------------------- @app.route("/support-request-submit", methods=["POST"]) @login_required def support_request_submit(): """Accept a support request, store it as JSON, and email it.""" body = request.get_json(silent=True) or {} name = body.get("name", "").strip() email = body.get("email", "").strip() description = body.get("description", "").strip() sr_datetime = body.get("datetime", "").strip() active_tool = body.get("active_tool", "").strip() app_state = body.get("app_state", "").strip() send_to = body.get("send_to", "").strip() or "icast.voyagetech.com@email.autotask.net" if not description: return jsonify(error="Issue description is required."), 400 user = session.get("user", {}) if not email: email = user.get("email", "") if not name: name = user.get("name", "Unknown") try: SUPPORT_DIR.mkdir(parents=True, exist_ok=True) except OSError as exc: return jsonify(error=f"Cannot create support request directory: {exc}"), 500 timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", name)[:40].strip().replace(" ", "_") filename = f"SR_{timestamp}_{safe_name}.json" out_path = SUPPORT_DIR / filename record = { "name": name, "email": email, "description": description, "datetime": sr_datetime or _datetime_type.now().isoformat(), "active_tool": active_tool, "app_state_json": app_state, "send_to": send_to, "submitted_at": _datetime_type.now().isoformat(), } try: out_path.write_text(json.dumps(record, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save support request: {exc}"), 500 # Send email via Microsoft Graph API if configured email_sent = False email_error = "" if AZURE_MAIL_FROM: try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" in token_result: email_body = ( f"Support Request from {name} ({email})\n" f"Date/Time: {sr_datetime}\n" f"Active Tool: {active_tool}\n\n" f"Issue Description:\n{description}\n\n" f"Application State:\n{app_state}\n\n" f"Reference: {filename}" ) graph_body = { "message": { "subject": f"Togen Support Request — {name} — {sr_datetime}", "body": { "contentType": "Text", "content": email_body, }, "from": { "emailAddress": {"address": AZURE_MAIL_FROM}, }, "replyTo": [ {"emailAddress": {"address": email}}, ], "toRecipients": [ {"emailAddress": {"address": send_to}}, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code in (200, 202): email_sent = True else: detail = resp.json().get("error", {}).get("message", resp.text[:200]) email_error = f"Graph API error: {detail}" else: email_error = "Failed to acquire mail token" except Exception as exc: email_error = str(exc) result = {"filename": filename, "email_sent": email_sent} if email_error: result["email_error"] = email_error return jsonify(result) @app.route("/feature-request-submit", methods=["POST"]) @login_required def feature_request_submit(): """Accept a feature request, store it as JSON, and email it.""" body = request.get_json(silent=True) or {} name = body.get("name", "").strip() email = body.get("email", "").strip() description = body.get("description", "").strip() beneficiaries = body.get("beneficiaries", "").strip() send_to = body.get("send_to", "").strip() or "icast.voyagetech.com@email.autotask.net" if not description: return jsonify(error="Feature description is required."), 400 user = session.get("user", {}) if not email: email = user.get("email", "") if not name: name = user.get("name", "Unknown") try: FEATURE_DIR.mkdir(parents=True, exist_ok=True) except OSError as exc: return jsonify(error=f"Cannot create feature request directory: {exc}"), 500 timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", name)[:40].strip().replace(" ", "_") filename = f"FR_{timestamp}_{safe_name}.json" out_path = FEATURE_DIR / filename record = { "name": name, "email": email, "description": description, "beneficiaries": beneficiaries, "send_to": send_to, "submitted_at": _datetime_type.now().isoformat(), } try: out_path.write_text(json.dumps(record, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save feature request: {exc}"), 500 # Send email via Microsoft Graph API if configured email_sent = False email_error = "" if AZURE_MAIL_FROM: try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" in token_result: email_body = ( f"Feature Request from {name} ({email})\n\n" f"Feature Description:\n{description}\n\n" f"Who may use or benefit from this feature:\n{beneficiaries}\n\n" f"Reference: {filename}" ) graph_body = { "message": { "subject": f"Togen Feature Request — {name}", "body": { "contentType": "Text", "content": email_body, }, "from": { "emailAddress": {"address": AZURE_MAIL_FROM}, }, "replyTo": [ {"emailAddress": {"address": email}}, ], "toRecipients": [ {"emailAddress": {"address": send_to}}, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code in (200, 202): email_sent = True else: detail = resp.json().get("error", {}).get("message", resp.text[:200]) email_error = f"Graph API error: {detail}" else: email_error = "Failed to acquire mail token" except Exception as exc: email_error = str(exc) result = {"filename": filename, "email_sent": email_sent} if email_error: result["email_error"] = email_error return jsonify(result) @app.route("/it-help-desk-submit", methods=["POST"]) @login_required def it_help_desk_submit(): """Accept an IT Help Desk ticket, store it, and email icast.voyagetech.com@email.autotask.net as the user.""" if not _wor_type_enabled_for("ithelpdesk", session["user"]): return jsonify(error="This work order type is not currently enabled."), 403 body = request.get_json(silent=True) or {} name = body.get("name", "").strip() email = body.get("email", "").strip() description = body.get("description", "").strip() related = body.get("related", []) if not isinstance(related, list): related = [] related = [str(r).strip() for r in related if str(r).strip()] if not description: return jsonify(error="Issue description is required."), 400 user = session.get("user", {}) if not email: email = user.get("email", "") if not name: name = user.get("name", "Unknown") send_to = "icast.voyagetech.com@email.autotask.net" try: SUPPORT_DIR.mkdir(parents=True, exist_ok=True) except OSError as exc: return jsonify(error=f"Cannot create request directory: {exc}"), 500 timestamp = _datetime_type.now().strftime("%Y-%m-%d_%H%M%S") safe_name = re.sub(r"[^\w\s\-]", "", name)[:40].strip().replace(" ", "_") filename = f"IT_{timestamp}_{safe_name}.json" out_path = SUPPORT_DIR / filename record = { "name": name, "email": email, "description": description, "related": related, "send_to": send_to, "submitted_at": _datetime_type.now().isoformat(), } try: out_path.write_text(json.dumps(record, indent=2)) except OSError as exc: return jsonify(error=f"Failed to save request: {exc}"), 500 related_line = ", ".join(related) if related else "(none selected)" # Send the email AS the submitting user so the ticketing software attributes # it to the requester. Falls back to the shared mailbox if the user's email # is unavailable (DVI-413). email_sent = False email_error = "" send_as = email or AZURE_MAIL_FROM if AZURE_MAIL_FROM and send_as: try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" in token_result: email_body = ( f"IT Help Desk ticket from {name} ({email})\n" f"Issue Related To: {related_line}\n\n" f"Issue Description:\n{description}\n\n" f"Reference: {filename}" ) graph_body = { "message": { "subject": f"IT Help Desk Request — {name}", "body": { "contentType": "Text", "content": email_body, }, "from": { "emailAddress": {"address": send_as}, }, "toRecipients": [ {"emailAddress": {"address": send_to}}, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{send_as}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code in (200, 202): email_sent = True else: detail = resp.json().get("error", {}).get("message", resp.text[:200]) email_error = f"Graph API error: {detail}" else: email_error = "Failed to acquire mail token" except Exception as exc: email_error = str(exc) result = {"filename": filename, "email_sent": email_sent} if email_error: result["email_error"] = email_error return jsonify(result) # --------------------------------------------------------------------------- # Routes — Project Builder # --------------------------------------------------------------------------- _SP_CSV_HEADERS = [ "tranId", "itemLine_custom:Structure", "itemLine_item", "itemLine_quantity", "itemLine_salesPrice", "itemLine_weight", "Item_Size", "Part Reference", "itemLine_description", "itemLine_custom:Stack-It PART - DrawingImage", "itemLine_custom:Submittal Drawing link", "itemLine_custom:QR Label", ] def _sp_next_project_number(): """Compute the next project number for today: T + YYYYMMDD + 2-digit seq.""" today = _datetime_type.now().strftime("%Y%m%d") prefix = f"T{today}" max_seq = 0 for search_dir in (SHARED_DIR, PROJECTBUILDER_DIR): if not search_dir.is_dir(): continue for entry in search_dir.iterdir(): if not entry.is_dir(): continue name = entry.name idx = name.find(prefix) if idx == -1: continue tail = name[idx + len(prefix):] digits = "" for ch in tail: if ch.isdigit(): digits += ch else: break if digits: max_seq = max(max_seq, int(digits)) return f"{prefix}{max_seq + 1:02d}" @app.route("/project-builder-next-number", methods=["GET"]) @takeoff_workflow_required def project_builder_next_number(): return jsonify({"number": _sp_next_project_number()}) @app.route("/project-builder-check", methods=["POST"]) @takeoff_workflow_required def project_builder_check(): data = request.get_json(silent=True) or {} name = data.get("folder_name", data.get("project_name", "")).strip() if not name or not _SAFE_JOB_RE.match(name) or ".." in name: return jsonify({"error": "Invalid folder name"}), 400 exists = (SHARED_DIR / name).is_dir() return jsonify({"exists": exists}) def _sp_filename_part(value): """Sanitize a project number / tranId for use in a CSV filename.""" return re.sub(r"[^A-Za-z0-9._-]+", "-", str(value).strip()).strip("-._") def _sp_csv_filename(project_number, rows, folder_name): """CSV name convention: -.csv (DVI-323).""" tran_id = "" for row in rows: if isinstance(row, list) and row and str(row[0]).strip(): tran_id = str(row[0]).strip() break num_part = _sp_filename_part(project_number) tran_part = _sp_filename_part(tran_id) if num_part and tran_part: return f"{num_part}-{tran_part}.csv" return f"{folder_name}.csv" @app.route("/project-builder-submit", methods=["POST"]) @takeoff_workflow_required def project_builder_submit(): data = request.get_json(silent=True) or {} project_name = data.get("project_name", "").strip() customer_name = data.get("customer_name", "").strip() vendor_project_code = data.get("vendor_project_code", "").strip() folder_name = data.get("folder_name", "").strip() project_number = data.get("project_number", "").strip() products = data.get("products", []) rows = data.get("rows", []) editing_folder = data.get("editing_folder", "").strip() import_source_folder = data.get("import_source_folder", "").strip() if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 if editing_folder and (not _SAFE_JOB_RE.match(editing_folder) or ".." in editing_folder): return jsonify({"error": "Invalid folder name"}), 400 if import_source_folder and (not _SAFE_JOB_RE.match(import_source_folder) or ".." in import_source_folder): return jsonify({"error": "Invalid folder name"}), 400 if not project_name: return jsonify({"error": "Project name is required"}), 400 if not rows: return jsonify({"error": "No product rows provided"}), 400 project_number = project_number or _sp_next_project_number() clean_rows = [row for row in rows if isinstance(row, list) and len(row) == len(_SP_CSV_HEADERS)] if not clean_rows: return jsonify({"error": "No product rows provided"}), 400 csv_filename = _sp_csv_filename(project_number, clean_rows, folder_name) if editing_folder: return _sp_update_existing_project( editing_folder, folder_name, csv_filename, clean_rows, customer_name, project_name, vendor_project_code, project_number, products, ) if import_source_folder: return _sp_import_project( import_source_folder, folder_name, csv_filename, clean_rows, customer_name, project_name, vendor_project_code, project_number, products, ) try: PROJECTBUILDER_DIR.mkdir(parents=True, exist_ok=True) except PermissionError: return jsonify({"error": "Server cannot write to temp directory. Contact admin."}), 500 tmp_folder = PROJECTBUILDER_DIR / folder_name tmp_folder.mkdir(parents=True, exist_ok=True) csv_path = tmp_folder / csv_filename metadata = { "customer_name": customer_name, "project_name": project_name, "vendor_project_code": vendor_project_code, "project_number": project_number, "folder_name": folder_name, "created_at": _datetime_type.now().isoformat(), "products": products, } meta_path = tmp_folder / "project_metadata.json" try: with open(csv_path, "w", newline="", encoding="utf-8") as fh: writer = csv.writer(fh) writer.writerow(_SP_CSV_HEADERS) writer.writerows(clean_rows) with open(meta_path, "w", encoding="utf-8") as fh: json.dump(metadata, fh, indent=2) _pb_move_temp_uploads(tmp_folder) dest = SHARED_DIR / folder_name if dest.exists(): shutil.rmtree(str(dest)) SHARED_DIR.mkdir(parents=True, exist_ok=True) shutil.copytree(str(tmp_folder), str(dest)) except PermissionError as exc: return jsonify({"error": f"Permission denied: {exc}"}), 500 except OSError as exc: return jsonify({"error": f"File system error: {exc}"}), 500 finally: shutil.rmtree(str(tmp_folder), ignore_errors=True) if _is_history_capture_enabled("project-builder"): record_history( "project-builder", "created", actor=(session.get("user") or {}).get("name"), summary=f"Project created: {project_name} ({folder_name})", ref=folder_name, detail={"project_number": project_number, "rows": len(clean_rows)}, ) return jsonify({"folder": folder_name, "csv": csv_filename}) def _sp_preserve_url_columns(source_csvs, clean_rows): """Carry forward cols J/K/L (DrawingImage / Submittal Drawing link / QR Label, indices 9-11) from existing CSV rows onto clean_rows so user edits in the builder don't wipe Bridge-written links. Matches first on (tranId, structure, item, part ref); falls back to (structure, item) for unmatched rows (DVI-323/DVI-387).""" entries = [] for old_csv in source_csvs: try: with open(old_csv, newline="", encoding="utf-8-sig") as fh: reader = csv.reader(fh) next(reader, None) for old_row in reader: if len(old_row) < 12: continue urls = [old_row[9], old_row[10], old_row[11]] if not any(u.strip() for u in urls): continue entries.append({ "key": tuple((old_row[i] or "").strip() for i in (0, 1, 2, 7)), "fb": tuple((old_row[i] or "").strip() for i in (1, 2)), "urls": urls, "used": False, }) except (OSError, csv.Error): continue def _take(field, value): for entry in entries: if not entry["used"] and entry[field] == value: entry["used"] = True return entry["urls"] return None unmatched = [] for row in clean_rows: key = tuple(str(row[i] or "").strip() for i in (0, 1, 2, 7)) urls = _take("key", key) if urls: row[9], row[10], row[11] = urls else: unmatched.append(row) for row in unmatched: fb = tuple(str(row[i] or "").strip() for i in (1, 2)) urls = _take("fb", fb) if urls: row[9], row[10], row[11] = urls def _sp_import_project(source_folder, folder_name, csv_filename, clean_rows, customer_name, project_name, vendor_project_code, project_number, products): """Import an externally-built (CSV-only, no metadata) project as a NEW project: copy its drawings, write a fresh CSV carrying over the J/K/L URLs, and write project_metadata.json — leaving the source folder untouched. The target folder name must be unique; never edit/overwrite an external project in place (DVI-387).""" src = SHARED_DIR / source_folder if not src.is_dir(): return jsonify({"error": "Source project folder not found"}), 404 dest = SHARED_DIR / folder_name if dest.exists(): return jsonify({"error": f'A project folder named "{folder_name}" already exists.'}), 409 source_csvs = sorted(f for f in src.iterdir() if f.is_file() and f.suffix.lower() == ".csv") _sp_preserve_url_columns(source_csvs, clean_rows) try: PROJECTBUILDER_DIR.mkdir(parents=True, exist_ok=True) except PermissionError: return jsonify({"error": "Server cannot write to temp directory. Contact admin."}), 500 tmp_folder = PROJECTBUILDER_DIR / folder_name shutil.rmtree(str(tmp_folder), ignore_errors=True) tmp_folder.mkdir(parents=True, exist_ok=True) try: for item in src.iterdir(): if item.is_file(): if item.suffix.lower() == ".csv" or item.name == "project_metadata.json": continue shutil.copy2(str(item), str(tmp_folder / item.name)) elif item.is_dir(): shutil.copytree(str(item), str(tmp_folder / item.name)) with open(tmp_folder / csv_filename, "w", newline="", encoding="utf-8") as fh: writer = csv.writer(fh) writer.writerow(_SP_CSV_HEADERS) writer.writerows(clean_rows) metadata = { "customer_name": customer_name, "project_name": project_name, "vendor_project_code": vendor_project_code, "project_number": project_number, "folder_name": folder_name, "created_at": _datetime_type.now().isoformat(), "imported_from": source_folder, "products": products, } with open(tmp_folder / "project_metadata.json", "w", encoding="utf-8") as fh: json.dump(metadata, fh, indent=2) _pb_move_temp_uploads(tmp_folder) if dest.exists(): return jsonify({"error": f'A project folder named "{folder_name}" already exists.'}), 409 SHARED_DIR.mkdir(parents=True, exist_ok=True) shutil.copytree(str(tmp_folder), str(dest)) except PermissionError as exc: return jsonify({"error": f"Permission denied: {exc}"}), 500 except OSError as exc: return jsonify({"error": f"File system error: {exc}"}), 500 finally: shutil.rmtree(str(tmp_folder), ignore_errors=True) if _is_history_capture_enabled("project-builder"): record_history( "project-builder", "imported", actor=(session.get("user") or {}).get("name"), summary=f"Project imported: {project_name} ({folder_name}) from {source_folder}", ref=folder_name, detail={"project_number": project_number, "rows": len(clean_rows), "source": source_folder}, ) return jsonify({"folder": folder_name, "csv": csv_filename}) def _sp_update_existing_project(editing_folder, folder_name, csv_filename, clean_rows, customer_name, project_name, vendor_project_code, project_number, products): """Update an opened project in place: rewrite its CSV (preserving the J/K/L drawing URLs already written by the Take-off Bridge), refresh project_metadata.json, and leave every other file (drawing PDFs) untouched (DVI-323).""" src_folder = SHARED_DIR / editing_folder if not src_folder.is_dir(): return jsonify({"error": "Project folder not found"}), 404 dest = SHARED_DIR / folder_name try: if editing_folder != folder_name: if dest.exists(): shutil.rmtree(str(dest)) shutil.move(str(src_folder), str(dest)) existing_csvs = sorted(f for f in dest.iterdir() if f.is_file() and f.suffix.lower() == ".csv") _sp_preserve_url_columns(existing_csvs, clean_rows) with open(dest / csv_filename, "w", newline="", encoding="utf-8") as fh: writer = csv.writer(fh) writer.writerow(_SP_CSV_HEADERS) writer.writerows(clean_rows) for old_csv in existing_csvs: if old_csv.name != csv_filename: old_csv.unlink() state_path = dest / "state.json" if state_path.is_file(): try: state = json.loads(state_path.read_text(encoding="utf-8")) if isinstance(state, dict) and state.get("csv_filename") != csv_filename: state["csv_filename"] = csv_filename state_path.write_text(json.dumps(state, indent=2), encoding="utf-8") except (json.JSONDecodeError, OSError): pass meta_path = dest / "project_metadata.json" metadata = {} if meta_path.is_file(): try: with open(meta_path, "r", encoding="utf-8") as fh: existing_meta = json.load(fh) if isinstance(existing_meta, dict): metadata = existing_meta except (json.JSONDecodeError, OSError): pass metadata.update({ "customer_name": customer_name, "project_name": project_name, "vendor_project_code": vendor_project_code, "project_number": project_number, "folder_name": folder_name, "products": products, }) metadata.setdefault("created_at", _datetime_type.now().isoformat()) metadata["updated_at"] = _datetime_type.now().isoformat() with open(meta_path, "w", encoding="utf-8") as fh: json.dump(metadata, fh, indent=2) _pb_move_temp_uploads(dest) except PermissionError as exc: return jsonify({"error": f"Permission denied: {exc}"}), 500 except OSError as exc: return jsonify({"error": f"File system error: {exc}"}), 500 if _is_history_capture_enabled("project-builder"): record_history( "project-builder", "edited", actor=(session.get("user") or {}).get("name"), summary=f"Project edited: {project_name} ({folder_name})", ref=folder_name, detail={"project_number": project_number, "rows": len(clean_rows)}, ) return jsonify({"folder": folder_name, "csv": csv_filename}) @app.route("/project-builder-list-projects", methods=["GET"]) @takeoff_workflow_required def project_builder_list_projects(): folders = [] if SHARED_DIR.is_dir(): for entry in sorted(SHARED_DIR.iterdir(), key=lambda e: e.stat().st_mtime, reverse=True): if entry.is_dir() and not entry.name.startswith("."): folders.append({ "name": entry.name, "has_meta": (entry / "project_metadata.json").is_file(), }) return jsonify({"folders": folders}) # Drawing-link columns on the take-off CSV (DVI-314): J=shop (idx 9), K=submittal # (idx 10). NO new columns — extra metadata lives in project_metadata.json. _SP_DRAWING_COL = {"shop": 9, "submittal": 10} _SP_DRAWING_EXTS = ( ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".tif", ".tiff", ".doc", ".docx", ".xls", ".xlsx", ".dwg", ".dxf", ) def _sp_row_key(row): """4-tuple identity for a CSV/product row: tranId, structure, item, part ref (cols A,B,C,H / idx 0,1,2,7) — matches _sp_preserve_url_columns.""" return tuple((row[i].strip() if len(row) > i else "") for i in (0, 1, 2, 7)) def _sp_overlay_drawing_urls(products, csv_rows): """Copy shop/submittal URLs from the CSV rows onto the metadata products list so the builder's row model surfaces drawing links on load. Matches first on the row key (tranId, structure, item, partRef); falls back to positional alignment for unmatched products (DVI-658).""" def key(d): return tuple( str(d.get(k, "") or "").strip() for k in ("tranId", "structure", "item", "partRef") ) pool = [{ "key": key(r), "shopUrl": r.get("shopUrl", ""), "submittalUrl": r.get("submittalUrl", ""), "used": False, } for r in csv_rows] for i, prod in enumerate(products): if not isinstance(prod, dict): continue match = None pk = key(prod) for entry in pool: if not entry["used"] and entry["key"] == pk: match = entry break if match is None and i < len(pool) and not pool[i]["used"]: match = pool[i] if match is not None: match["used"] = True prod["shopUrl"] = match["shopUrl"] prod["submittalUrl"] = match["submittalUrl"] def _sp_locate_csv(project_folder): """Return (csv_path, None) for a project folder's CSV — state.json csv_filename first, else the sole *.csv — or (None, error_message). Mirrors scs-save-to-project.""" state_path = project_folder / "state.json" if state_path.is_file(): try: state = json.loads(state_path.read_text(encoding="utf-8")) name = state.get("csv_filename") if name and (project_folder / name).is_file(): return project_folder / name, None except (json.JSONDecodeError, OSError): pass csvs = sorted(f for f in project_folder.iterdir() if f.is_file() and f.suffix.lower() == ".csv") if len(csvs) == 1: return csvs[0], None if len(csvs) > 1: return None, "Multiple CSV files found — project records not updated" return None, "No CSV found for this project" def _sp_match_data_row(rows, row_index, key): """Index (0-based, among data rows) of the CSV row a drawing belongs to. Prefers row_index when its key matches the supplied key, else first key match. Returns None when no row matches.""" data = rows[1:] has_key = any(key) if isinstance(row_index, int) and 0 <= row_index < len(data): if not has_key or _sp_row_key(data[row_index]) == key: return row_index if has_key: for i, r in enumerate(data): if _sp_row_key(r) == key: return i return None def _sp_update_meta_product(project_folder, data_index, file_type, url): """Best-effort: reflect a drawing add/clear on the matching products entry in project_metadata.json so metadata stays consistent with the CSV.""" meta_path = project_folder / "project_metadata.json" if not meta_path.is_file(): return try: meta = json.loads(meta_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return products = meta.get("products") if not isinstance(products, list) or not (0 <= data_index < len(products)): return prod = products[data_index] if not isinstance(prod, dict): return prod["shopUrl" if file_type == "shop" else "submittalUrl"] = url try: meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8") except OSError: pass @app.route("/project-builder-upload-drawing", methods=["POST"]) @takeoff_workflow_required def project_builder_upload_drawing(): """Attach (or replace) a shop/submittal drawing on a Project Builder product row: store the file in the existing project folder and write its public URL into CSV col J (shop) or K (submittal). Mirrors Take-off /upload-file scoped to a SHARED_DIR project folder (DVI-658).""" if "file" not in request.files: return jsonify({"error": "No file provided"}), 400 f = request.files["file"] if not f.filename: return jsonify({"error": "No file provided"}), 400 ext = os.path.splitext(f.filename)[1].lower() if ext not in _SP_DRAWING_EXTS: return jsonify({"error": "Unsupported file type"}), 400 folder_name = (request.form.get("folder") or "").strip() file_type = (request.form.get("file_type") or "").strip() if file_type not in _SP_DRAWING_COL: return jsonify({"error": "Invalid file_type"}), 400 if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 project_folder = (SHARED_DIR / folder_name).resolve() if not str(project_folder).startswith(str(SHARED_DIR.resolve()) + os.sep) or not project_folder.is_dir(): return jsonify({"error": "Project folder not found"}), 404 csv_path, warn = _sp_locate_csv(project_folder) if csv_path is None: return jsonify({"error": warn}), 400 save_name = secure_filename(f.filename) if not save_name: return jsonify({"error": "Invalid file name"}), 400 row_index = request.form.get("row_index", type=int) key = tuple((request.form.get(k) or "").strip() for k in ("tranId", "structure", "item", "partRef")) with open(csv_path, newline="", encoding="utf-8-sig") as fh: rows = list(csv.reader(fh)) matched = _sp_match_data_row(rows, row_index, key) if matched is None: return jsonify({"error": "Could not match the product row in the CSV"}), 409 f.save(str(project_folder / save_name)) col = _SP_DRAWING_COL[file_type] _qr_base = _get_qr_base_url().rstrip("/") url = f"{_qr_base}/{quote(folder_name, safe='')}/{quote(save_name, safe='')}" row = rows[matched + 1] while len(row) < 12: row.append("") row[col] = url with open(csv_path, "w", newline="", encoding="utf-8") as fh: csv.writer(fh).writerows(rows) _sp_update_meta_product(project_folder, matched, file_type, url) if _is_history_capture_enabled("project-builder"): record_history( "project-builder", "drawing-attached", actor=(session.get("user") or {}).get("name"), summary=f"Drawing attached to {folder_name} ({file_type}): {save_name}", ref=folder_name, detail={"file_type": file_type, "filename": save_name}, ) return jsonify({"ok": True, "url": url, "filename": save_name}) @app.route("/project-builder-detach-drawing", methods=["POST"]) @takeoff_workflow_required def project_builder_detach_drawing(): """Clear a shop/submittal drawing link from a Project Builder product row (CSV col J/K + metadata). The file itself is left in the folder, mirroring Take-off /detach-file (DVI-658).""" data = request.get_json(silent=True) or {} folder_name = (data.get("folder") or "").strip() file_type = (data.get("file_type") or "").strip() if file_type not in _SP_DRAWING_COL: return jsonify({"error": "Invalid file_type"}), 400 if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder"}), 400 project_folder = (SHARED_DIR / folder_name).resolve() if not str(project_folder).startswith(str(SHARED_DIR.resolve()) + os.sep) or not project_folder.is_dir(): return jsonify({"error": "Project folder not found"}), 404 csv_path, warn = _sp_locate_csv(project_folder) if csv_path is None: return jsonify({"error": warn}), 400 row_index = data.get("row_index") if not isinstance(row_index, int): try: row_index = int(row_index) except (TypeError, ValueError): row_index = None key = tuple((str(data.get(k) or "")).strip() for k in ("tranId", "structure", "item", "partRef")) with open(csv_path, newline="", encoding="utf-8-sig") as fh: rows = list(csv.reader(fh)) matched = _sp_match_data_row(rows, row_index, key) if matched is None: return jsonify({"error": "Could not match the product row in the CSV"}), 409 col = _SP_DRAWING_COL[file_type] row = rows[matched + 1] if len(row) > col: row[col] = "" with open(csv_path, "w", newline="", encoding="utf-8") as fh: csv.writer(fh).writerows(rows) _sp_update_meta_product(project_folder, matched, file_type, "") return jsonify({"ok": True}) # ===== Project Builder file upload + OCR discovery (DVI-661) ================= # Files OCR'd for project-info discovery; everything else is stored/moved but # never read (per approved plan: store any type, OCR only PDF + images). _PB_OCR_EXTS = (".pdf", ".png", ".jpg", ".jpeg") # iCast transaction id: ##-####[letter] where the leading 2 digits are a recent # year (24-27) and the optional trailing letter is a b-version (DVI-661). The # dash is optional in source text; we always normalize back to ##-####[letter]. _PB_TRANID_RE = re.compile(r"\b(2[4-7])-?(\d{4})([A-Za-z]?)\b") def _parse_tran_id(text: str) -> str: """Pull the first iCast tranId out of arbitrary text and normalize it to ##-####[letter]. Returns "" when no match. The trailing letter's case is preserved (b-versions are conventionally lowercase).""" m = _PB_TRANID_RE.search(text or "") if not m: return "" return f"{m.group(1)}-{m.group(2)}{m.group(3) or ''}" def _pb_image_to_pdf(img_path: Path, dest_pdf: str) -> None: """Convert a raster image to a one-page PDF so the title-block OCR path (which expects a PDF) can read it. Raises on failure — callers guard.""" import fitz # noqa: PLC0415 — PyMuPDF, already a dependency doc = fitz.open(str(img_path)) try: pdf_bytes = doc.convert_to_pdf() finally: doc.close() Path(dest_pdf).write_bytes(pdf_bytes) def _pb_discover_fields(path: Path) -> dict: """OCR one PDF/image for Project Builder prefill. Returns any subset of {customer_name, project_name, tranId}. Images are converted to a one-page PDF first. Best-effort: never raises, returns {} for non-OCR types.""" ext = path.suffix.lower() if ext not in _PB_OCR_EXTS: return {} pdf_path = path tmp_pdf = None try: if ext != ".pdf": fd, tmp_pdf = tempfile.mkstemp(suffix=".pdf") os.close(fd) try: _pb_image_to_pdf(path, tmp_pdf) pdf_path = Path(tmp_pdf) except Exception: # noqa: BLE001 — unreadable image → nothing to OCR return {} out = {} labeled = _suggest_fields_from_pdf( pdf_path, ["customer_name", "project_name", "job_number"] ) if labeled.get("customer_name"): out["customer_name"] = labeled["customer_name"] if labeled.get("project_name"): out["project_name"] = labeled["project_name"] try: from ocr_engine import extract_title_block_text raw = extract_title_block_text(pdf_path) except Exception: # noqa: BLE001 raw = "" tran = (_parse_tran_id(raw) or _parse_tran_id(labeled.get("job_number", "")) or _parse_tran_id(path.name)) if tran: out["tranId"] = tran return out finally: if tmp_pdf: try: os.unlink(tmp_pdf) except OSError: pass def _pb_aggregate_discovery(paths) -> dict: """OCR each path in order; first non-empty value per field wins. Stops once all three target fields are filled to keep large folders responsive.""" suggested = {} for p in paths: for k, v in _pb_discover_fields(p).items(): if v and not suggested.get(k): suggested[k] = v if all(suggested.get(k) for k in ("customer_name", "project_name", "tranId")): break return suggested def _pb_session_id() -> str: """Stable per-browser-session id for scoping temp uploads. Stored in the Flask session so a user's staged files never collide with another user's.""" sid = session.get("pb_upload_sid") if not sid or not re.match(r"^[a-f0-9]{32}$", str(sid)): sid = uuid.uuid4().hex session["pb_upload_sid"] = sid return sid def _pb_temp_dir(create: bool = False): """Resolve PROJECTBUILDER_DIR/_uploads//, rejecting traversal. Returns the Path (creating it when asked) or None on a bad session id.""" base = (PROJECTBUILDER_DIR / "_uploads").resolve() d = (base / _pb_session_id()).resolve() if not str(d).startswith(str(base) + os.sep): return None if create: d.mkdir(parents=True, exist_ok=True) return d def _pb_temp_listing() -> list: """List this session's staged uploads: [{name, ocr}] where ocr flags a PDF/image that OCR Discovery can read.""" d = _pb_temp_dir(create=False) out = [] if d and d.is_dir(): for item in sorted(d.iterdir()): if item.is_file(): out.append({"name": item.name, "ocr": item.suffix.lower() in _PB_OCR_EXTS}) return out def _pb_move_temp_uploads(dest_folder: Path) -> None: """Move this session's staged uploads into dest_folder (on Submit), then clear the session temp dir. Files whose name already exists in dest are skipped (e.g. the CSV/metadata written by the submit handler).""" src = _pb_temp_dir(create=False) if not src or not src.is_dir(): return try: for item in src.iterdir(): if not item.is_file(): continue target = dest_folder / item.name if target.exists(): continue shutil.move(str(item), str(target)) except OSError as exc: app.logger.warning("PB temp upload move failed for %s: %s", dest_folder, exc) shutil.rmtree(str(src), ignore_errors=True) session.pop("pb_upload_sid", None) @app.route("/project-builder/upload-temp", methods=["POST"]) @takeoff_workflow_required def project_builder_upload_temp(): """Stage one or more uploaded files in a session-scoped temp dir. They are not in the final folder yet — Submit moves them into SHARED_DIR//. Any file type is accepted (DVI-661).""" files = request.files.getlist("files") or request.files.getlist("file") if not files: return jsonify(error="No files provided"), 400 d = _pb_temp_dir(create=True) if d is None: return jsonify(error="Invalid session"), 400 for f in files: if not f or not f.filename: continue name = secure_filename(f.filename) if not name: continue target = (d / name).resolve() if not str(target).startswith(str(d) + os.sep): continue f.save(str(target)) return jsonify(ok=True, files=_pb_temp_listing()) @app.route("/project-builder/remove-temp", methods=["POST"]) @takeoff_workflow_required def project_builder_remove_temp(): """Remove one staged upload from the session temp dir (DVI-661).""" data = request.get_json(silent=True) or {} name = secure_filename((data.get("filename") or "").strip()) if not name: return jsonify(error="Invalid filename"), 400 d = _pb_temp_dir(create=False) if d and d.is_dir(): target = (d / name).resolve() if str(target).startswith(str(d) + os.sep) and target.is_file(): try: target.unlink() except OSError: pass return jsonify(ok=True, files=_pb_temp_listing()) @app.route("/project-builder/temp-files", methods=["GET"]) @takeoff_workflow_required def project_builder_temp_files(): """List the current session's staged uploads (DVI-661).""" return jsonify(files=_pb_temp_listing()) @app.route("/project-builder/discover-temp", methods=["POST"]) @takeoff_workflow_required def project_builder_discover_temp(): """OCR the session's staged PDF/image uploads and return suggested, user-editable Project Information values (DVI-661 A.1).""" d = _pb_temp_dir(create=False) paths = [] if d and d.is_dir(): paths = [p for p in sorted(d.iterdir()) if p.is_file() and p.suffix.lower() in _PB_OCR_EXTS] return jsonify(fields=_pb_aggregate_discovery(paths)) @app.route("/project-builder/discover-folder", methods=["POST"]) @takeoff_workflow_required def project_builder_discover_folder(): """OCR the documents in an opened/imported project folder to discover Project Information — primarily the Customer/Contractor name (DVI-661 B).""" data = request.get_json(silent=True) or {} folder_name = (data.get("folder_name") or "").strip() if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify(error="Invalid folder"), 400 project_folder = (SHARED_DIR / folder_name).resolve() if not str(project_folder).startswith(str(SHARED_DIR.resolve()) + os.sep) \ or not project_folder.is_dir(): return jsonify(error="Project folder not found"), 404 paths = [p for p in sorted(project_folder.iterdir()) if p.is_file() and p.suffix.lower() in _PB_OCR_EXTS] return jsonify(fields=_pb_aggregate_discovery(paths)) # Assistant document correlation (DVI-662, Feature D1). Caps keep the seeded # prompt's token cost bounded regardless of how many files are in scope. _PB_CORR_MAX_DOCS = 12 _PB_CORR_SNIPPET_CHARS = 600 def _pb_doc_snippet(path: Path) -> str: """Short, whitespace-collapsed title-block text snippet for one document, capped at _PB_CORR_SNIPPET_CHARS. Only OCR-able types (PDF/image) yield a snippet; everything else returns "". Best-effort: never raises.""" ext = path.suffix.lower() if ext not in _PB_OCR_EXTS: return "" pdf_path = path tmp_pdf = None try: if ext != ".pdf": fd, tmp_pdf = tempfile.mkstemp(suffix=".pdf") os.close(fd) try: _pb_image_to_pdf(path, tmp_pdf) pdf_path = Path(tmp_pdf) except Exception: # noqa: BLE001 — unreadable image → no snippet return "" try: from ocr_engine import extract_title_block_text raw = extract_title_block_text(pdf_path) or "" except Exception: # noqa: BLE001 raw = "" raw = re.sub(r"\s+", " ", raw).strip() return raw[:_PB_CORR_SNIPPET_CHARS] finally: if tmp_pdf: try: os.unlink(tmp_pdf) except OSError: pass def _pb_correlation_paths(folder_name: str) -> list: """Collect the documents currently in scope for correlation: this session's temp uploads plus the opened/imported project folder's files. OCR-able docs are listed first so they win the _PB_CORR_MAX_DOCS budget for snippets.""" paths = [] seen = set() def _add(p: Path): key = p.name.lower() if key not in seen and p.is_file(): seen.add(key) paths.append(p) temp_dir = _pb_temp_dir(create=False) folder_paths = [] if folder_name and _SAFE_JOB_RE.match(folder_name) and ".." not in folder_name: project_folder = (SHARED_DIR / folder_name).resolve() if str(project_folder).startswith(str(SHARED_DIR.resolve()) + os.sep) \ and project_folder.is_dir(): folder_paths = [p for p in sorted(project_folder.iterdir()) if p.is_file()] temp_paths = [] if temp_dir and temp_dir.is_dir(): temp_paths = [p for p in sorted(temp_dir.iterdir()) if p.is_file()] candidates = temp_paths + folder_paths # OCR-able types first so they keep their snippets within the doc cap. for p in sorted(candidates, key=lambda p: p.suffix.lower() not in _PB_OCR_EXTS): _add(p) return paths def _pb_build_correlation_prompt(paths: list) -> str: """Build the Assistant seed prompt: a correlation instruction followed by a markdown list of each in-scope document with a short OCR snippet.""" lines = [ "I'm setting up a project in our Project Builder and need help " "understanding how the following documents relate to each other.", "", "For each document, help me determine:", "- which is the **submittal** package vs **shop drawings** vs other reference docs,", "- which **product line** each drawing maps to (e.g. by transaction/part number),", "- the overall **project identity** (customer/contractor, project name, transaction number).", "", "## Documents in scope", ] for i, p in enumerate(paths, 1): lines.append("") lines.append(f"### {i}. {p.name}") snippet = _pb_doc_snippet(p) if snippet: lines.append("```") lines.append(snippet) lines.append("```") else: lines.append("_(no readable text extracted — non-OCR file type or empty)_") return "\n".join(lines) @app.route("/project-builder/correlate-docs", methods=["POST"]) @takeoff_workflow_required def project_builder_correlate_docs(): """Build a correlation seed prompt from the documents currently in scope — this session's temp uploads plus any opened/imported project folder — for the Assistant chat surface (DVI-662, Feature D1). Returns the prompt text and a doc count; the frontend pre-seeds the existing Assistant with it. No LLM call or auto-apply happens here.""" data = request.get_json(silent=True) or {} folder_name = (data.get("folder_name") or "").strip() paths = _pb_correlation_paths(folder_name)[:_PB_CORR_MAX_DOCS] if not paths: return jsonify(error="No documents in scope to correlate. Upload files or open a project first."), 400 return jsonify(prompt=_pb_build_correlation_prompt(paths), doc_count=len(paths)) @app.route("/project-builder-load-project", methods=["POST"]) @takeoff_workflow_required def project_builder_load_project(): data = request.get_json(silent=True) or {} folder_name = data.get("folder_name", "").strip() if not folder_name or not _SAFE_JOB_RE.match(folder_name) or ".." in folder_name: return jsonify({"error": "Invalid folder name"}), 400 folder_path = SHARED_DIR / folder_name if not folder_path.is_dir(): return jsonify({"error": "Folder not found"}), 404 result = {"folder_name": folder_name, "meta": None, "csv_rows": []} meta_path = folder_path / "project_metadata.json" if meta_path.is_file(): try: with open(meta_path, "r", encoding="utf-8") as fh: result["meta"] = json.load(fh) except (json.JSONDecodeError, OSError): pass csv_path = None for f in folder_path.iterdir(): if f.suffix.lower() == ".csv" and f.is_file(): csv_path = f break if csv_path: try: with open(csv_path, "r", encoding="utf-8") as fh: reader = csv.DictReader(fh) for row in reader: mapped = {} for field in _SP_CSV_HEADERS: for sp_field in [ {"csv": "tranId", "id": "tranId"}, {"csv": "itemLine_custom:Structure", "id": "structure"}, {"csv": "itemLine_item", "id": "item"}, {"csv": "itemLine_quantity", "id": "quantity"}, {"csv": "itemLine_salesPrice", "id": "salesPrice"}, {"csv": "itemLine_weight", "id": "weight"}, {"csv": "Item_Size", "id": "itemSize"}, {"csv": "Part Reference", "id": "partRef"}, {"csv": "itemLine_description", "id": "description"}, ]: if sp_field["csv"] == field and field in row: mapped[sp_field["id"]] = row[field] # Surface the J/K drawing links (preserved in the CSV but never # mapped onto the row model) so the builder can show the drawing # indicator and manage attachments (DVI-658). mapped["shopUrl"] = (row.get(_SP_CSV_HEADERS[9]) or "").strip() mapped["submittalUrl"] = (row.get(_SP_CSV_HEADERS[10]) or "").strip() if any(v for v in mapped.values()): result["csv_rows"].append(mapped) except (OSError, csv.Error): pass # The opened project prefers meta.products as its row model; overlay the # authoritative CSV J/K links onto it so drawing state survives a reload. if isinstance(result["meta"], dict) and isinstance(result["meta"].get("products"), list): _sp_overlay_drawing_urls(result["meta"]["products"], result["csv_rows"]) # When project_metadata.json is absent, meta stays None and the client prefills # from the shared parseFolderNameForProject() JS parser — keep folder-name # parsing in that single implementation. return jsonify(result) # Top-bar global Search (DVI-534). Backend half: matches take-off project folders # by name. Tool/navigation matches are computed client-side from the TOOLS array # (visibility/membership already live there). Project results are gated to # take-off / admin users; everyone else simply gets an empty project list and # still searches tools client-side. @app.route("/search", methods=["GET"]) @login_required def global_search(): q = (request.args.get("q") or "").strip() projects = [] if q: u = session.get("user", {}) if (u.get("takeoff_workflow") or u.get("togen_admin")) and SHARED_DIR.is_dir(): needle = q.lower() for entry in sorted( SHARED_DIR.iterdir(), key=lambda e: e.stat().st_mtime, reverse=True ): if not entry.is_dir() or entry.name.startswith("."): continue if needle in entry.name.lower(): projects.append({ "name": entry.name, "has_meta": (entry / "project_metadata.json").is_file(), }) if len(projects) >= 20: break return jsonify({"query": q, "projects": projects}) _TUTORIALS_DIR = Path("/var/www/html/togen/tutorials") @app.route("/togen/tutorials/manifest.json") @login_required def tutorials_manifest(): """Return the list of available tutorial videos from the tutorials directory.""" manifest_path = _TUTORIALS_DIR / "manifest.json" if manifest_path.is_file(): try: data = json.loads(manifest_path.read_text()) return jsonify(data) except Exception: return jsonify([]) tutorials = [] if _TUTORIALS_DIR.is_dir(): for f in sorted(_TUTORIALS_DIR.iterdir()): if f.suffix.lower() in (".mp4", ".webm", ".mov"): tutorials.append({ "title": f.stem.replace("-", " ").replace("_", " ").title(), "url": f"/togen/tutorials/{quote(f.name)}", "thumbnail": "", }) return jsonify(tutorials) # --------------------------------------------------------------------------- # File Browser routes # --------------------------------------------------------------------------- _FB_ROOT = SHARED_DIR # /var/www/html/shared/ def _fb_safe_path(rel: str) -> Path | None: """Resolve a relative path under _FB_ROOT; return None if it escapes.""" cleaned = rel.strip().strip("/") if ".." in cleaned.split("/") or cleaned.startswith("/"): return None resolved = (_FB_ROOT / cleaned).resolve() if not str(resolved).startswith(str(_FB_ROOT.resolve())): return None return resolved @app.route("/fb/browse") @takeoff_workflow_required def fb_browse(): """List files and folders in the given directory under SHARED_DIR.""" rel = request.args.get("path", "") target = _fb_safe_path(rel) if rel else _FB_ROOT if target is None or not target.is_dir(): return jsonify({"error": "Invalid directory"}), 400 entries = [] try: for item in sorted(target.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): if item.name.startswith("."): continue stat = item.stat() entries.append({ "name": item.name, "is_dir": item.is_dir(), "size": stat.st_size if item.is_file() else 0, "modified": _datetime_type.fromtimestamp(stat.st_mtime).isoformat(), }) except PermissionError: return jsonify({"error": "Permission denied"}), 403 rel_display = str(target.relative_to(_FB_ROOT)) if rel_display == ".": rel_display = "" return jsonify({"path": rel_display, "entries": entries}) # --------------------------------------------------------------------------- # File Browser folder watchers (DVI-471) # # A watcher is a per-user automation that emails the user (via SSO address) # whenever anything under a chosen folder in the shared Projects tree changes. # Watcher definitions live in the user's settings JSON under ``fb_watchers``; # the background loop snapshots each folder every 15 minutes and emails on diff. # --------------------------------------------------------------------------- def _fb_watchers_from_settings(settings: dict) -> list: """Return the list of watcher dicts stored in a user's settings.""" watchers = settings.get("fb_watchers") return watchers if isinstance(watchers, list) else [] def _fb_parse_expiry(value): """Parse a watcher termination date (``YYYY-MM-DD``) into a ``date``. Returns ``None`` for empty/invalid input. The watcher stays active through the termination date itself and expires once the day has fully passed. """ if not value: return None try: return _datetime_type.fromisoformat(str(value).strip()[:10]).date() except (ValueError, TypeError): return None def _fb_watcher_expired(watcher: dict) -> bool: """True when the watcher carries a termination date that has passed.""" exp = _fb_parse_expiry(watcher.get("expires_at")) return exp is not None and _datetime_type.now().date() > exp # Public site URL used for 1-click email action links (matches News emails). _FB_APP_URL = "https://togen.icastinc.com" def _fb_watcher_remove_link(wid: str) -> str: """1-click link that opens Togen and prompts to remove a specific watcher.""" return f"{_FB_APP_URL}/?fb_remove_watcher={quote(str(wid))}" def _fb_current_email() -> str: return (session.get("user") or {}).get("email", "") def _fb_snapshot_folder(folder: Path) -> dict: """Build a {relative_path: 'mtime:size'} map of every file under folder. Dotfiles are skipped (mirrors the browse listing). Used to detect adds, removals, and modifications between watcher cycles. """ snap = {} if not folder.is_dir(): return snap try: for fpath in folder.rglob("*"): if not fpath.is_file() or fpath.name.startswith("."): continue try: st = fpath.stat() except OSError: continue snap[str(fpath.relative_to(folder))] = f"{int(st.st_mtime)}:{st.st_size}" except OSError: pass return snap def _fb_load_watcher_state() -> dict: if not FB_WATCHER_STATE_FILE.is_file(): return {} try: data = json.loads(FB_WATCHER_STATE_FILE.read_text()) return data if isinstance(data, dict) else {} except (json.JSONDecodeError, OSError): return {} def _fb_save_watcher_state(state: dict) -> None: try: FB_WATCHER_STATE_FILE.write_text(json.dumps(state, indent=2)) except OSError: app.logger.exception("Failed to persist fb_watcher_state.json") @app.route("/fb/watchers") @takeoff_workflow_required def fb_watchers_list(): """Return the current user's folder watchers.""" email = _fb_current_email() return jsonify(watchers=_fb_watchers_from_settings(_load_user_settings(email))) @app.route("/fb/watchers", methods=["POST"]) @takeoff_workflow_required def fb_watchers_add(): """Create a folder watcher for the current user on a folder in the tree.""" email = _fb_current_email() body = request.get_json(silent=True) or {} rel = (body.get("path") or "").strip().strip("/") if not rel: return jsonify(error="A folder path is required."), 400 target = _fb_safe_path(rel) if target is None or not target.is_dir(): return jsonify(error="Folder not found."), 400 expires_raw = (body.get("expires_at") or "").strip() if expires_raw and _fb_parse_expiry(expires_raw) is None: return jsonify(error="Invalid expiration date."), 400 settings = _load_user_settings(email) watchers = list(_fb_watchers_from_settings(settings)) if any(w.get("path") == rel for w in watchers): return jsonify(error="That folder is already being watched."), 409 wid = uuid.uuid4().hex watchers.append({ "id": wid, "path": rel, "label": Path(rel).name or rel, "enabled": True, "created_at": _datetime_type.now().isoformat(), "expires_at": expires_raw or None, }) settings["fb_watchers"] = watchers try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save watcher: {exc}"), 500 # Seed the snapshot so the first cycle does not fire on pre-existing files. state = _fb_load_watcher_state() state[wid] = { "owner": email, "path": rel, "snapshot": _fb_snapshot_folder(target), "last_checked": _datetime_type.now().isoformat(), } _fb_save_watcher_state(state) return jsonify(ok=True, watchers=watchers) @app.route("/fb/watchers//toggle", methods=["POST"]) @takeoff_workflow_required def fb_watchers_toggle(wid): """Enable/disable email notifications for a watcher (keeps the watcher).""" email = _fb_current_email() body = request.get_json(silent=True) or {} enabled = bool(body.get("enabled")) settings = _load_user_settings(email) watchers = list(_fb_watchers_from_settings(settings)) found = False for w in watchers: if w.get("id") == wid: w["enabled"] = enabled found = True break if not found: return jsonify(error="Watcher not found."), 404 settings["fb_watchers"] = watchers try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save watcher: {exc}"), 500 return jsonify(ok=True, watchers=watchers) @app.route("/fb/watchers/", methods=["PUT"]) @takeoff_workflow_required def fb_watchers_update(wid): """Edit a watcher's termination date (set, change, or clear it).""" email = _fb_current_email() body = request.get_json(silent=True) or {} expires_raw = (body.get("expires_at") or "").strip() if expires_raw and _fb_parse_expiry(expires_raw) is None: return jsonify(error="Invalid expiration date."), 400 settings = _load_user_settings(email) watchers = list(_fb_watchers_from_settings(settings)) found = False for w in watchers: if w.get("id") == wid: w["expires_at"] = expires_raw or None found = True break if not found: return jsonify(error="Watcher not found."), 404 settings["fb_watchers"] = watchers try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save watcher: {exc}"), 500 return jsonify(ok=True, watchers=watchers) @app.route("/fb/watchers/", methods=["DELETE"]) @takeoff_workflow_required def fb_watchers_delete(wid): """Delete a watcher automation and its stored snapshot.""" email = _fb_current_email() settings = _load_user_settings(email) watchers = [w for w in _fb_watchers_from_settings(settings) if w.get("id") != wid] settings["fb_watchers"] = watchers try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to delete watcher: {exc}"), 500 state = _fb_load_watcher_state() if wid in state: state.pop(wid, None) _fb_save_watcher_state(state) return jsonify(ok=True, watchers=watchers) @app.route("/fb/upload", methods=["POST"]) @takeoff_workflow_required def fb_upload(): """Upload one or more files to the given directory. Extract ZIP files automatically.""" rel = request.form.get("path", "") target = _fb_safe_path(rel) if rel else _FB_ROOT if target is None or not target.is_dir(): return jsonify({"error": "Invalid directory"}), 400 files = request.files.getlist("files") if not files: return jsonify({"error": "No files provided"}), 400 uploaded_names = [] for f in files: if not f.filename: continue safe_name = secure_filename(f.filename) if not safe_name: continue dest = target / safe_name if safe_name.lower().endswith(".zip"): tmp_path = target / f"_tmp_{uuid.uuid4().hex}_{safe_name}" try: f.save(str(tmp_path)) with zipfile.ZipFile(str(tmp_path)) as zf: for member in zf.namelist(): member_dest = (target / member).resolve() if not str(member_dest).startswith(str(target.resolve())): continue zf.extractall(str(target)) uploaded_names.append(f"{safe_name} (extracted)") except zipfile.BadZipFile: f.seek(0) f.save(str(dest)) uploaded_names.append(safe_name) finally: if tmp_path.exists(): tmp_path.unlink() else: f.save(str(dest)) uploaded_names.append(safe_name) if uploaded_names and _is_history_capture_enabled("file-browser"): record_history( "file-browser", "upload", actor=(session.get("user") or {}).get("name"), summary=f"Uploaded {len(uploaded_names)} file(s) to {rel or '/'}", ref=rel or "/", detail={"count": len(uploaded_names), "dir": rel or "/"}, ) return jsonify({"uploaded": uploaded_names}) @app.route("/fb/download") @takeoff_workflow_required def fb_download(): """Download a file or folder. Folders are zipped on the fly.""" rel = request.args.get("path", "") if not rel: return jsonify({"error": "No path specified"}), 400 target = _fb_safe_path(rel) if target is None or not target.exists(): return jsonify({"error": "File not found"}), 404 if target.is_file(): if _is_history_capture_enabled("file-browser"): record_history( "file-browser", "download", actor=(session.get("user") or {}).get("name"), summary=f"Downloaded {rel}", ref=rel, detail={"type": "file"}, ) return send_file(str(target), as_attachment=True, download_name=target.name) if target.is_dir(): if target.resolve() == _FB_ROOT.resolve(): return jsonify({"error": "Cannot download root directory"}), 400 buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: for fpath in target.rglob("*"): if fpath.is_file() and not fpath.name.startswith("."): zf.write(str(fpath), str(fpath.relative_to(target))) buf.seek(0) if _is_history_capture_enabled("file-browser"): record_history( "file-browser", "download", actor=(session.get("user") or {}).get("name"), summary=f"Downloaded folder {rel} (zip)", ref=rel, detail={"type": "folder"}, ) return send_file(buf, as_attachment=True, download_name=f"{target.name}.zip", mimetype="application/zip") return jsonify({"error": "Unknown target type"}), 400 @app.route("/fb/delete", methods=["POST"]) @takeoff_workflow_required def fb_delete(): """Delete a file or folder.""" data = request.get_json(silent=True) or {} rel = data.get("path", "") if not rel: return jsonify({"error": "No path specified"}), 400 target = _fb_safe_path(rel) if target is None or not target.exists(): return jsonify({"error": "File not found"}), 404 if target.resolve() == _FB_ROOT.resolve(): return jsonify({"error": "Cannot delete root directory"}), 400 try: if target.is_dir(): shutil.rmtree(str(target)) else: target.unlink() except PermissionError: return jsonify({"error": "Permission denied"}), 403 if _is_history_capture_enabled("file-browser"): record_history( "file-browser", "delete", actor=(session.get("user") or {}).get("name"), summary=f"Deleted {rel}", ref=rel, ) return jsonify({"deleted": rel}) @app.route("/fb/rename", methods=["POST"]) @takeoff_workflow_required def fb_rename(): """Rename a file or folder.""" data = request.get_json(silent=True) or {} rel = data.get("path", "") new_name = data.get("new_name", "").strip() if not rel or not new_name: return jsonify({"error": "Missing path or new_name"}), 400 if "/" in new_name or "\\" in new_name or ".." in new_name: return jsonify({"error": "Invalid new name"}), 400 target = _fb_safe_path(rel) if target is None or not target.exists(): return jsonify({"error": "File not found"}), 404 if target.resolve() == _FB_ROOT.resolve(): return jsonify({"error": "Cannot rename root directory"}), 400 new_path = target.parent / secure_filename(new_name) if new_path.exists(): return jsonify({"error": "A file or folder with that name already exists"}), 409 try: target.rename(new_path) except PermissionError: return jsonify({"error": "Permission denied"}), 403 if _is_history_capture_enabled("file-browser"): record_history( "file-browser", "rename", actor=(session.get("user") or {}).get("name"), summary=f"Renamed {target.name} → {new_path.name}", ref=rel, ) return jsonify({"renamed": new_path.name}) @app.route("/fb/preview") @takeoff_workflow_required def fb_preview(): """Serve a file inline for preview.""" rel = request.args.get("path", "") if not rel: return "Missing path", 400 target = _fb_safe_path(rel) if target is None or not target.is_file(): return "File not found", 404 return send_file(str(target), download_name=target.name) # --------------------------------------------------------------------------- # Routes — News (DVI-267) # --------------------------------------------------------------------------- def _promote_scheduled_posts(): """Publish any scheduled posts whose publish_at has arrived. Scheduled posts carry status "scheduled" plus a UTC publish_at. On the first read at/after that time they flip to "published" and the subscriber email goes out — a daemon-free lazy publish driven by page loads. Returns the (possibly mutated) posts list, newest first. """ posts = _load_news_posts() now_str = _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") due = [p for p in posts if p.get("status") == "scheduled" and p.get("publish_at", "") <= now_str] if not due: return posts for post in due: post["status"] = "published" _save_news_posts(posts) subscribers = _load_subscribers() if subscribers and AZURE_MAIL_FROM: for post in due: _send_news_email(post, subscribers) return posts @app.route("/news/posts") @login_required def news_list(): """Return news posts, newest first. Scheduled posts are admin-only.""" posts = _promote_scheduled_posts() if not session.get("user", {}).get("togen_admin"): posts = [p for p in posts if p.get("status") != "scheduled"] return jsonify(posts=posts) @app.route("/news/posts", methods=["POST"]) @togen_admin_required def news_create(): """Create a news post, optionally scheduled for future publish. Admin only.""" body = request.get_json(silent=True) or {} title = body.get("title", "").strip() content = body.get("body", "").strip() publish_at_raw = (body.get("publish_at") or "").strip() if not title or not content: return jsonify(error="Title and body are required."), 400 now_str = _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") publish_at = "" if publish_at_raw: try: dt = _datetime_type.fromisoformat(publish_at_raw.replace("Z", "+00:00")) except ValueError: return jsonify(error="Invalid publish date/time."), 400 if dt.tzinfo is not None: dt = dt.astimezone(_dt_timezone.utc).replace(tzinfo=None) publish_at = dt.strftime("%Y-%m-%dT%H:%M:%SZ") scheduled = bool(publish_at) and publish_at > now_str user = session.get("user", {}) post = { "id": str(uuid.uuid4()), "title": title, "body": content, "author": user.get("name", "Unknown"), "author_email": user.get("email", ""), "date": publish_at if scheduled else now_str, "status": "scheduled" if scheduled else "published", } if scheduled: post["publish_at"] = publish_at posts = _load_news_posts() posts.insert(0, post) _save_news_posts(posts) if not scheduled: subscribers = _load_subscribers() if subscribers and AZURE_MAIL_FROM: _send_news_email(post, subscribers) return jsonify(ok=True, post=post) @app.route("/news/posts/", methods=["PUT"]) @togen_admin_required def news_update(post_id): """Edit a scheduled (not-yet-published) news post. Original poster only. Allows changing title, body, and the scheduled publish time before the post goes live. Published posts are immutable here. Only the user who authored the post may edit it. """ body = request.get_json(silent=True) or {} title = body.get("title", "").strip() content = body.get("body", "").strip() publish_at_raw = (body.get("publish_at") or "").strip() if not title or not content: return jsonify(error="Title and body are required."), 400 posts = _load_news_posts() post = next((p for p in posts if p.get("id") == post_id), None) if post is None: return jsonify(error="Post not found."), 404 if post.get("status") != "scheduled": return jsonify(error="Only scheduled posts can be edited."), 400 user = session.get("user", {}) user_email = (user.get("email") or "").lower() if (post.get("author_email") or "").lower() != user_email: return jsonify(error="Only the original poster can edit this post."), 403 now_str = _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") if not publish_at_raw: return jsonify(error="A scheduled publish date/time is required."), 400 try: dt = _datetime_type.fromisoformat(publish_at_raw.replace("Z", "+00:00")) except ValueError: return jsonify(error="Invalid publish date/time."), 400 if dt.tzinfo is not None: dt = dt.astimezone(_dt_timezone.utc).replace(tzinfo=None) publish_at = dt.strftime("%Y-%m-%dT%H:%M:%SZ") if publish_at <= now_str: return jsonify(error="Pick a publish time in the future."), 400 post["title"] = title post["body"] = content post["publish_at"] = publish_at post["date"] = publish_at _save_news_posts(posts) return jsonify(ok=True, post=post) @app.route("/news/posts/", methods=["DELETE"]) @togen_admin_required def news_delete(post_id): """Delete a news post. Admin only.""" posts = _load_news_posts() posts = [p for p in posts if p.get("id") != post_id] _save_news_posts(posts) return jsonify(ok=True) def _md_escape(text): """Escape HTML special chars so user content can't inject markup.""" return (text.replace("&", "&") .replace("<", "<") .replace(">", ">")) def _render_news_markdown(text): """Convert a lightweight Markdown subset to email-safe HTML. Supports: ATX headings (#, ##, ###), unordered (-, *) and ordered lists, blank-line-separated paragraphs with single newlines as
    , **bold**, *italic*, [text](url) links, and bare http(s) autolinks. All text is HTML-escaped first, so the source is untrusted-safe. Explicit block tags are emitted (rather than relying on white-space:pre-wrap) because Outlook ignores pre-wrap and collapses the post's spacing. """ link_style = "color:#0077A8;text-decoration:underline;" def _inline(s): s = _md_escape(s) placeholders = [] def _stash(tag_html): placeholders.append(tag_html) return "\x00{}\x00".format(len(placeholders) - 1) # Markdown links: [text](url) — stash so later passes don't touch them. def _md_link(m): return _stash('{}'.format( m.group(2), link_style, m.group(1))) s = re.sub(r"\[([^\]]+)\]\((https?://[^\s)]+)\)", _md_link, s) # Bare URL autolink (URLs already inside a stashed link are safe). def _bare_link(m): url = m.group(1) return _stash('{}'.format( url, link_style, url)) s = re.sub(r"(?=])(https?://[^\s<]+)", _bare_link, s) s = re.sub(r"\*\*([^*]+)\*\*", r"\1", s) s = re.sub(r"(?\1", s) for idx, html_frag in enumerate(placeholders): s = s.replace("\x00{}\x00".format(idx), html_frag) return s lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") parts = [] para = [] list_items = [] list_tag = None def flush_para(): if para: parts.append('

    ' + "
    ".join(para) + "

    ") para.clear() def flush_list(): nonlocal list_tag if list_items: items = "".join( '
  • {}
    • '.format(it) for it in list_items) parts.append('<{0} style="margin:0 0 14px;padding-left:22px;">{1}' .format(list_tag, items)) list_items.clear() list_tag = None for line in lines: stripped = line.strip() if not stripped: flush_para() flush_list() continue heading = re.match(r"^(#{1,3})\s+(.*)$", stripped) if heading: flush_para() flush_list() level = len(heading.group(1)) sizes = {1: "22px", 2: "18px", 3: "16px"} parts.append( '{2}'.format( level, sizes[level], _inline(heading.group(2)))) continue ul = re.match(r"^[-*]\s+(.*)$", stripped) ol = re.match(r"^\d+\.\s+(.*)$", stripped) if ul or ol: flush_para() want = "ul" if ul else "ol" if list_tag and list_tag != want: flush_list() list_tag = want list_items.append(_inline((ul or ol).group(1))) continue flush_list() para.append(_inline(stripped)) flush_para() flush_list() return "".join(parts) or '

     

    ' def _news_email_shell(title, meta_html, body_html, cta_html, footer_html): """Wrap content in a branded, Outlook-safe (table-based) HTML email.""" return ( '' '' '' '' '' '
    ' '' # Header bar '' # Body '' # Footer '' '
    ' 'togen' '' '
    News Update
    ' '
    ' '

    {title}

    ' '{meta}' '
    ' '{body}' '{cta}' '
    {footer}
    ' ).format(title=title, meta=meta_html, body=body_html, cta=cta_html, footer=footer_html) _NEWS_UNSUBSCRIBE_FOOTER = ( '
    ' '

    ' 'You are receiving this email because you subscribed to Togen News updates.
    ' 'Unsubscribe from future updates.' '

    ' ) def _send_news_email(post, subscribers): """Send a news post to all subscribers via Microsoft Graph API (fire-and-forget).""" def _do_send(): try: meta_html = ( '

    ' 'Posted by {author} on {date}

    '.format( author=_md_escape(post.get("author", "")), date=_md_escape(post.get("date", "")[:10])) ) cta_html = ( '' ) content = _news_email_shell( title=_md_escape(post["title"]), meta_html=meta_html, body_html=_render_news_markdown(post.get("body", "")), cta_html=cta_html, footer_html=_NEWS_UNSUBSCRIBE_FOOTER, ) notifications.send_html_email( list(subscribers), f"Togen News: {post['title']}", content) except Exception: pass threading.Thread(target=_do_send, daemon=True).start() def _send_subscribe_confirmation(email): """Send a subscription confirmation email (fire-and-forget).""" def _do_send(): try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: return body_html = ( '

    You will now receive ' 'email notifications when new posts are published on Togen.

    ' ) cta_html = ( '' ) content = _news_email_shell( title="You're subscribed to Togen News!", meta_html="", body_html=body_html, cta_html=cta_html, footer_html=_NEWS_UNSUBSCRIBE_FOOTER, ) graph_body = { "message": { "subject": "Togen News: Subscription Confirmed", "body": { "contentType": "HTML", "content": content, }, "toRecipients": [ {"emailAddress": {"address": email}}, ], }, "saveToSentItems": "false", } _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) except Exception: pass threading.Thread(target=_do_send, daemon=True).start() # --------------------------------------------------------------------------- # Routes — News Subscribers (DVI-267) # --------------------------------------------------------------------------- @app.route("/news/subscribers", methods=["GET"]) @login_required def news_subscribers_get(): """Check if the current user is subscribed.""" user = session.get("user", {}) email = user.get("email", "").lower() subs = _load_subscribers() return jsonify(subscribed=email in [s.lower() for s in subs]) @app.route("/news/subscribe", methods=["POST"]) @login_required def news_subscribe(): """Subscribe an email to news notifications.""" body = request.get_json(silent=True) or {} email = body.get("email", "").strip().lower() if not email or "@" not in email: return jsonify(error="Valid email address is required."), 400 subs = _load_subscribers() existing = [s.lower() for s in subs] is_new = email not in existing if is_new: subs.append(email) _save_subscribers(subs) if is_new and AZURE_MAIL_FROM: _send_subscribe_confirmation(email) return jsonify(ok=True, subscribed=True) @app.route("/news/unsubscribe", methods=["POST"]) @login_required def news_unsubscribe(): """Unsubscribe an email from news notifications.""" body = request.get_json(silent=True) or {} email = body.get("email", "").strip().lower() subs = _load_subscribers() subs = [s for s in subs if s.lower() != email] _save_subscribers(subs) return jsonify(ok=True, subscribed=False) # --------------------------------------------------------------------------- # Routes — Public Version (DVI-267) # --------------------------------------------------------------------------- @app.route("/togen-admin/public-version") @login_required def togen_admin_public_version_get(): """Return the current public version string.""" return jsonify(version=_load_public_version()) @app.route("/togen-admin/public-version", methods=["POST"]) @togen_admin_required def togen_admin_public_version_post(): """Update the public version string. Admin only.""" body = request.get_json(silent=True) or {} version = body.get("version", "").strip() if not version: return jsonify(error="Version text is required."), 400 try: PUBLIC_VERSION_FILE.write_text(version + "\n") except OSError as exc: return jsonify(error=f"Failed to save version: {exc}"), 500 return jsonify(ok=True, version=version) # --------------------------------------------------------------------------- # Routes — About content (DVI-412) # --------------------------------------------------------------------------- @app.route("/togen-admin/about") @login_required def togen_admin_about_get(): """Return the editable About popup text (Markdown). Any logged-in user.""" return jsonify(content=_load_about_content()) @app.route("/togen-admin/about", methods=["POST"]) @togen_admin_required def togen_admin_about_post(): """Update the About popup text. Admin only.""" body = request.get_json(silent=True) or {} content = (body.get("content") or "").strip() if not content: return jsonify(error="About text is required."), 400 try: _save_about_content(content) except OSError as exc: return jsonify(error=f"Failed to save About text: {exc}"), 500 return jsonify(ok=True, content=content) @app.route("/togen-admin/site-text") @login_required def togen_admin_site_text_get(): """Return the editable header title/tagline. Any logged-in user.""" return jsonify(_load_site_text()) @app.route("/togen-admin/site-text", methods=["POST"]) @togen_admin_required def togen_admin_site_text_post(): """Update the header title/tagline. Admin only.""" body = request.get_json(silent=True) or {} title = (body.get("title") or "").strip() tagline = (body.get("tagline") or "").strip() if not title: return jsonify(error="Title is required."), 400 if not tagline: return jsonify(error="Tagline is required."), 400 try: _save_site_text(title, tagline) except OSError as exc: return jsonify(error=f"Failed to save title: {exc}"), 500 return jsonify(ok=True, title=title, tagline=tagline) # --------------------------------------------------------------------------- # Routes — Sync service stop/start (DVI-412) # stop/start invoke `sudo systemctl` against rclone-bisync.service; the state # file is kept as fallback / desired-state for dev environments where systemctl # is unavailable. Requires sudoers entry for the web-server user (see services/). # --------------------------------------------------------------------------- @app.route("/togen-admin/sync-service/status") @togen_admin_required def togen_admin_sync_service_status(): return jsonify(running=_load_sync_service_running()) @app.route("/togen-admin/sync-service/stop", methods=["POST"]) @togen_admin_required def togen_admin_sync_service_stop(): try: subprocess.run( ["sudo", "systemctl", "stop", _BISYNC_UNIT], check=True, capture_output=True, timeout=15, ) except Exception: pass # best-effort; state file tracks intent for UI _save_sync_service_running(False) running = _load_sync_service_running() return jsonify(ok=True, running=running) @app.route("/togen-admin/sync-service/start", methods=["POST"]) @togen_admin_required def togen_admin_sync_service_start(): try: subprocess.run( ["sudo", "systemctl", "start", _BISYNC_UNIT], check=True, capture_output=True, timeout=15, ) except Exception: pass # best-effort; state file tracks intent for UI _save_sync_service_running(True) running = _load_sync_service_running() return jsonify(ok=True, running=running) # --------------------------------------------------------------------------- # Routes — Shared URL Prefix (DVI-271) # --------------------------------------------------------------------------- @app.route("/togen-admin/shared-url-prefix") @togen_admin_required def togen_admin_url_prefix_get(): """Return the current shared URL prefix and available options.""" return jsonify( prefix=_load_shared_url_prefix(), options=SHARED_URL_PREFIX_OPTIONS, ) @app.route("/togen-admin/shared-url-prefix", methods=["POST"]) @togen_admin_required def togen_admin_url_prefix_post(): """Update the shared URL prefix. Admin only.""" body = request.get_json(silent=True) or {} prefix = (body.get("prefix") or "").strip().rstrip("/") valid_values = [opt["value"] for opt in SHARED_URL_PREFIX_OPTIONS] if prefix not in valid_values: return jsonify(error="Invalid URL prefix selected."), 400 _save_shared_url_prefix(prefix) return jsonify(ok=True, prefix=prefix) # --------------------------------------------------------------------------- # Routes — Branding (DVI-357) # --------------------------------------------------------------------------- @app.route("/togen-admin/branding") @togen_admin_required def togen_admin_branding_get(): """Return current branding settings plus which corporate assets exist.""" settings = _load_branding_settings() return jsonify( logo_mode=settings["logo_mode"], bg_mode=settings["bg_mode"], bg_opacity_light=settings["bg_opacity_light"], bg_opacity_dark=settings["bg_opacity_dark"], tile_columns=settings["tile_columns"], tile_side_padding=settings["tile_side_padding"], has_logo=CORPORATE_LOGO_FILE.is_file(), has_bg_light=CORPORATE_BG_LIGHT_FILE.is_file(), has_bg_dark=CORPORATE_BG_DARK_FILE.is_file(), ) @app.route("/togen-admin/branding", methods=["POST"]) @togen_admin_required def togen_admin_branding_post(): """Update branding logo_mode and/or bg_mode. Admin only.""" body = request.get_json(silent=True) or {} settings = _load_branding_settings() if "logo_mode" in body: if body["logo_mode"] not in BRANDING_LOGO_MODES: return jsonify(error="Invalid logo mode."), 400 settings["logo_mode"] = body["logo_mode"] if "bg_mode" in body: if body["bg_mode"] not in BRANDING_BG_MODES: return jsonify(error="Invalid background mode."), 400 settings["bg_mode"] = body["bg_mode"] for key in ("bg_opacity_light", "bg_opacity_dark"): if key in body: opacity = _coerce_bg_opacity(body[key]) if opacity is None: return jsonify(error="Opacity must be an integer between 40 and 100."), 400 settings[key] = opacity if "tile_columns" in body: tile_columns = _coerce_tile_columns(body["tile_columns"]) if tile_columns is None: return jsonify(error='Tile columns must be 3-9 or "unlimited".'), 400 settings["tile_columns"] = tile_columns if "tile_side_padding" in body: tile_side_padding = _coerce_tile_side_padding(body["tile_side_padding"]) if tile_side_padding is None: return jsonify(error="Side padding must be an integer between 0 and 40."), 400 settings["tile_side_padding"] = tile_side_padding _save_branding_settings(settings) return jsonify(ok=True, **settings) @app.route("/togen-admin/branding/logo.svg") def togen_admin_branding_logo(): """Serve the corporate logo SVG. No auth (referenced by ); 404 if absent.""" if not CORPORATE_LOGO_FILE.is_file(): abort(404) return send_file(CORPORATE_LOGO_FILE, mimetype="image/svg+xml") @app.route("/togen-admin/branding/bg-light.svg") def togen_admin_branding_bg_light(): """Serve the corporate light-theme background SVG. No auth; 404 if absent.""" if not CORPORATE_BG_LIGHT_FILE.is_file(): abort(404) return send_file(CORPORATE_BG_LIGHT_FILE, mimetype="image/svg+xml") @app.route("/togen-admin/branding/bg-dark.svg") def togen_admin_branding_bg_dark(): """Serve the corporate dark-theme background SVG. No auth; 404 if absent.""" if not CORPORATE_BG_DARK_FILE.is_file(): abort(404) return send_file(CORPORATE_BG_DARK_FILE, mimetype="image/svg+xml") @app.route("/togen-admin/subscribers") @togen_admin_required def togen_admin_subscribers_list(): """Return all news subscriber emails. Admin only.""" subs = _load_subscribers() return jsonify(subscribers=subs) # --------------------------------------------------------------------------- # Routes — Printing (DVI-548) # --------------------------------------------------------------------------- def _load_printing_settings(): """Return default printer settings: {document_printer, label_printer, enabled}. ``enabled`` (DVI-764) gates whether the Print system is offered to non-admin users. It defaults to ``True`` so existing deployments keep working; an admin can clear the checkbox to hide/disable Print for regular users (Togen admins always retain access).""" settings = {"document_printer": "", "label_printer": "", "enabled": True} if PRINTING_SETTINGS_FILE.is_file(): try: saved = json.loads(PRINTING_SETTINGS_FILE.read_text()) if isinstance(saved, dict): if isinstance(saved.get("document_printer"), str): settings["document_printer"] = saved["document_printer"] if isinstance(saved.get("label_printer"), str): settings["label_printer"] = saved["label_printer"] if isinstance(saved.get("enabled"), bool): settings["enabled"] = saved["enabled"] except (json.JSONDecodeError, OSError): pass return settings def _print_system_enabled(): """True when the Print system is enabled for regular users (DVI-764).""" return _load_printing_settings().get("enabled", True) def _user_can_print(user): """Whether ``user`` may use the Print system (DVI-764). Togen admins always may; everyone else only when the system is enabled.""" if user and user.get("togen_admin"): return True return _print_system_enabled() _WINSTON_DEFAULTS = { "draft_mailbox": "sales@icastinc.com", "clickup_api_key": "", "clickup_list_id": "", "azure_client_id": "", "azure_tenant_id": "", "azure_client_secret": "", "onedrive_user": "", "takeoffs_folder": "Infrastructure Precast Inc_ - Takeoffs", "report_to": [], "report_cc": [], "shared_quotes_location": "", "shared_quote_takeoff_location": "", } def _load_winston_settings() -> dict: settings = dict(_WINSTON_DEFAULTS) if WINSTON_SETTINGS_FILE.is_file(): try: saved = json.loads(WINSTON_SETTINGS_FILE.read_text()) if isinstance(saved, dict): for k, default in _WINSTON_DEFAULTS.items(): if k in saved and type(saved[k]) is type(default): settings[k] = saved[k] except (json.JSONDecodeError, OSError): pass return settings def _save_winston_settings(settings: dict): WINSTON_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) def _available_printers(): """Printers published by active Print agents, split for the two dropdowns. Returns ``{"documents": [...], "labels": [...]}`` — sorted, de-duplicated lists of ``published_name`` strings drawn from included printers flagged ``for_documents`` / ``for_labels``. A printer with no published_name falls back to its raw_name so it is still selectable. Only printers belonging to agents in the ``active`` status are offered (pending/revoked agents are not trusted). (DVI-668 / DVI-657 P2) """ docs, labels = {}, {} conn = _print_db() try: rows = conn.execute( "SELECT p.raw_name, p.published_name, p.for_documents, p.for_labels" " FROM print_printers p" " JOIN print_agents a ON a.machine_id = p.machine_id" " WHERE p.included = 1 AND a.status = 'active'" ).fetchall() finally: conn.close() for r in rows: name = (r["published_name"] or "").strip() or (r["raw_name"] or "").strip() if not name: continue key = name.lower() if r["for_documents"]: docs.setdefault(key, name) if r["for_labels"]: labels.setdefault(key, name) return { "documents": sorted(docs.values(), key=str.lower), "labels": sorted(labels.values(), key=str.lower), } # Sentinel persisted for a user dropdown left on "Use Default" — the effective # resolver treats it (and a blank value) as "fall back to the global default". PRINT_USER_DEFAULT = "__default__" def _user_printing_overrides(email): """Return the user's stored printer overrides, defaulting to the sentinel. Stored under ``user_settings/.json`` -> ``printing: {document_printer, label_printer}``. Missing keys default to ``__default__`` (use global).""" stored = (_load_user_settings(email) or {}).get("printing") or {} doc = stored.get("document_printer") lbl = stored.get("label_printer") return { "document_printer": doc if isinstance(doc, str) and doc else PRINT_USER_DEFAULT, "label_printer": lbl if isinstance(lbl, str) and lbl else PRINT_USER_DEFAULT, } def _effective_default_printers(email): """Resolve the effective default printers for a user (DVI-668 / P2). A user override (a non-sentinel, non-empty published_name) wins; otherwise the global default from ``printing_settings.json`` is used. Shared helper for the print-dispatch (P3) and tool Print buttons (P4).""" glob = _load_printing_settings() user = _user_printing_overrides(email) def _pick(key): chosen = user.get(key) if chosen and chosen != PRINT_USER_DEFAULT: return chosen return glob.get(key, "") return { "document_printer": _pick("document_printer"), "label_printer": _pick("label_printer"), } @app.route("/togen-admin/printing") @togen_admin_required def togen_admin_printing_get(): """Return default printer settings + available printers. Admin only.""" settings = _load_printing_settings() return jsonify( document_printer=settings["document_printer"], label_printer=settings["label_printer"], enabled=settings["enabled"], available_printers=_available_printers(), ) @app.route("/user-settings/printing") @login_required def user_settings_printing_get(): """Return the user's printer overrides + global defaults + the picker lists. Each dropdown offers a "Use Default" sentinel (``__default__``) plus the published printers for that kind. (DVI-668 / DVI-657 P2)""" email = session["user"].get("email", "") glob = _load_printing_settings() return jsonify( available_printers=_available_printers(), global_defaults={ "document_printer": glob["document_printer"], "label_printer": glob["label_printer"], }, user=_user_printing_overrides(email), default_sentinel=PRINT_USER_DEFAULT, ) @app.route("/user-settings/printing", methods=["POST"]) @login_required def user_settings_printing_post(): """Persist the user's default-printer overrides. The sentinel ``__default__`` (or a blank value) clears the override so the global default applies.""" email = session["user"].get("email", "") body = request.get_json(silent=True) or {} avail = _available_printers() allowed = { "document_printer": {n.lower() for n in avail["documents"]}, "label_printer": {n.lower() for n in avail["labels"]}, } settings = _load_user_settings(email) printing = dict(settings.get("printing") or {}) for key in ("document_printer", "label_printer"): if key not in body: continue val = body[key] if val in (None, "", PRINT_USER_DEFAULT): printing.pop(key, None) continue if not isinstance(val, str): return jsonify(error=f"Invalid value for {key}."), 400 if val.lower() not in allowed[key]: return jsonify(error=f"{val} is not an available printer."), 400 printing[key] = val if printing: settings["printing"] = printing else: settings.pop("printing", None) try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify(ok=True, user=_user_printing_overrides(email)) @app.route("/togen-admin/printing", methods=["POST"]) @togen_admin_required def togen_admin_printing_post(): """Save default document/label printer selections. Admin only.""" body = request.get_json(silent=True) or {} settings = _load_printing_settings() if "document_printer" in body: settings["document_printer"] = str(body["document_printer"] or "") if "label_printer" in body: settings["label_printer"] = str(body["label_printer"] or "") if "enabled" in body: settings["enabled"] = bool(body["enabled"]) PRINTING_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) return jsonify(ok=True, enabled=settings["enabled"]) # --------------------------------------------------------------------------- # Winston settings — DVI-808 # Admin: shared quotes location; User: personal quotes location. # --------------------------------------------------------------------------- @app.route("/togen-admin/quotes-location") @togen_admin_required def togen_admin_quotes_location_get(): """Return the admin-configured shared quotes paths (completed + take-offs).""" settings = _load_winston_settings() return jsonify( shared_quotes_location=settings.get("shared_quotes_location", ""), shared_quote_takeoff_location=settings.get("shared_quote_takeoff_location", ""), ) @app.route("/togen-admin/quotes-location", methods=["POST"]) @togen_admin_required def togen_admin_quotes_location_post(): """Persist the shared quotes paths (completed + take-offs). Admin only.""" body = request.get_json(silent=True) or {} settings = _load_winston_settings() for key in ("shared_quotes_location", "shared_quote_takeoff_location"): if key in body: val = body[key] if not isinstance(val, str): return jsonify(error=f"{key} must be a string."), 400 settings[key] = val.strip() try: _save_winston_settings(settings) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify( ok=True, shared_quotes_location=settings.get("shared_quotes_location", ""), shared_quote_takeoff_location=settings.get("shared_quote_takeoff_location", ""), ) @app.route("/user-settings/documents") @login_required def user_settings_documents_get(): """Return the user's personal documents settings (quotes paths).""" email = session["user"].get("email", "") settings = _load_user_settings(email) docs = settings.get("documents") or {} return jsonify( quotes_location=docs.get("quotes_location", ""), quote_takeoff_location=docs.get("quote_takeoff_location", ""), ) @app.route("/user-settings/documents", methods=["POST"]) @login_required def user_settings_documents_post(): """Persist the user's personal documents settings (quotes paths).""" email = session["user"].get("email", "") body = request.get_json(silent=True) or {} settings = _load_user_settings(email) docs = dict(settings.get("documents") or {}) for key in ("quotes_location", "quote_takeoff_location"): if key in body: val = body[key] if not isinstance(val, str): return jsonify(error=f"{key} must be a string."), 400 if val.strip(): docs[key] = val.strip() else: docs.pop(key, None) if docs: settings["documents"] = docs else: settings.pop("documents", None) try: _save_user_settings(email, settings) except OSError as exc: return jsonify(error=f"Failed to save settings: {exc}"), 500 return jsonify( ok=True, quotes_location=docs.get("quotes_location", ""), quote_takeoff_location=docs.get("quote_takeoff_location", ""), ) @app.route("/winston/quote-paths") @login_required def winston_quote_paths_get(): """Aggregate read-only view of all four resolved quote path values for the current user. Returns: shared_completed — shared_quotes_location from admin settings shared_quote_takeoff — shared_quote_takeoff_location from admin settings personal_completed — quotes_location from the user's documents settings personal_quote_takeoff — quote_takeoff_location from the user's documents settings """ shared = _load_winston_settings() email = session["user"].get("email", "") user_docs = (_load_user_settings(email).get("documents") or {}) return jsonify( shared_completed=shared.get("shared_quotes_location", ""), shared_quote_takeoff=shared.get("shared_quote_takeoff_location", ""), personal_completed=user_docs.get("quotes_location", ""), personal_quote_takeoff=user_docs.get("quote_takeoff_location", ""), ) # --------------------------------------------------------------------------- # Winston P4 — page UI backend (DVI-817) # Settings, Schedules, Runs-History, Run-now # All routes gated togen_admin_required (mirrors the tool's default-disabled # → admins-only registration). # --------------------------------------------------------------------------- _WINSTON_SCHEDULE_CADENCES = ("daily", "weekdays", "custom") _WINSTON_SCHEDULE_PROFILES = ("prev_biz", "today") _WINSTON_MAX_RUN_RECORDS = 100 # Use the shared lock exported from winston_scheduler so scheduler threads and # the run-now route both hold the same lock when writing winston_run_records.json. from togen import winston_scheduler as _winston_sched_mod _WINSTON_RUN_LOCK = _winston_sched_mod.run_records_lock _winston_active_runs: dict = {} # run_id → True while running def _load_winston_schedules() -> list: if WINSTON_SCHEDULES_FILE.is_file(): try: data = json.loads(WINSTON_SCHEDULES_FILE.read_text()) if isinstance(data, list): return [s for s in data if isinstance(s, dict)] except (json.JSONDecodeError, OSError): pass return [] def _save_winston_schedules(schedules: list): WINSTON_SCHEDULES_FILE.write_text(json.dumps(schedules, indent=2)) def _load_winston_run_records() -> list: if WINSTON_RUN_RECORDS_FILE.is_file(): try: data = json.loads(WINSTON_RUN_RECORDS_FILE.read_text()) if isinstance(data, list): return [r for r in data if isinstance(r, dict)] except (json.JSONDecodeError, OSError): pass return [] def _save_winston_run_records(records: list): WINSTON_RUN_RECORDS_FILE.write_text(json.dumps(records, indent=2)) def _append_winston_run_record(record: dict): """Thread-safe append + trim to _WINSTON_MAX_RUN_RECORDS, newest first.""" with _WINSTON_RUN_LOCK: records = _load_winston_run_records() records = [r for r in records if r.get("id") != record.get("id")] records.insert(0, record) records = records[:_WINSTON_MAX_RUN_RECORDS] _save_winston_run_records(records) def _winston_schedule_from_body(body: dict, existing: dict | None = None): """Build/validate a schedule dict. Returns (schedule, error).""" name = (body.get("name") or "").strip() if not name: return None, "Schedule name is required." if len(name) > 120: return None, "Schedule name is too long (max 120 characters)." cadence = (body.get("cadence") or "daily").strip() if cadence not in _WINSTON_SCHEDULE_CADENCES: return None, f"Invalid cadence; expected one of {', '.join(_WINSTON_SCHEDULE_CADENCES)}." custom_days = None if cadence == "custom": raw_days = body.get("custom_days") or [] if not isinstance(raw_days, list): return None, "custom_days must be a list." try: custom_days = sorted({int(d) for d in raw_days if 0 <= int(d) <= 6}) except (TypeError, ValueError): return None, "custom_days values must be integers 0–6 (Mon=0 … Sun=6)." if not custom_days: return None, "custom_days must have at least one day." try: hour = int(body.get("hour", 6)) if not (0 <= hour <= 23): raise ValueError except (TypeError, ValueError): return None, "hour must be an integer 0–23." try: minute = int(body.get("minute", 0)) if not (0 <= minute <= 59): raise ValueError except (TypeError, ValueError): return None, "minute must be an integer 0–59." profile = (body.get("profile") or "today").strip() from togen import winston_date_logic as _wdl valid_profiles = _wdl.valid_pipeline_ids() | set(_WINSTON_SCHEDULE_PROFILES) if profile not in valid_profiles: return None, "Invalid Date Logic; select an existing pipeline." now = _datetime_type.now().isoformat() schedule = { "id": (existing or {}).get("id") or str(uuid.uuid4()), "name": name, "cadence": cadence, "custom_days": custom_days, "hour": hour, "minute": minute, "profile": profile, "enabled": bool(body.get("enabled", True)), "created_at": (existing or {}).get("created_at") or now, "updated_at": now, } return schedule, None @app.route("/winston/config") @togen_admin_required def winston_config_get(): """Return Winston configuration status for the Settings tab.""" from togen import winston_graph as _wg settings = _load_winston_settings() # Resolve effective values: saved settings take precedence over env vars. azure_client_id = settings.get("azure_client_id", "") or _wg._AZ_CLIENT_ID azure_tenant_id = settings.get("azure_tenant_id", "") or _wg._AZ_TENANT azure_secret_set = bool(settings.get("azure_client_secret", "") or _wg._AZ_SECRET) clickup_list_id = settings.get("clickup_list_id", "") or os.environ.get("CLICKUP_LIST_ID", "").strip() onedrive_user = settings.get("onedrive_user", "") or _wg.WINSTON_ONEDRIVE_USER takeoffs_folder = ( settings.get("takeoffs_folder", "") or os.environ.get("WINSTON_TAKEOFFS_FOLDER", "") or "Infrastructure Precast Inc_ - Takeoffs" ) clickup_key_set = bool( os.environ.get("CLICKUP_API_KEY") or settings.get("clickup_api_key", "") ) azure_configured = bool(azure_client_id and azure_secret_set and azure_tenant_id) return jsonify( draft_mailbox=settings.get("draft_mailbox", ""), clickup_api_key_configured=clickup_key_set, clickup_list_id=clickup_list_id, azure_configured=azure_configured, azure_client_id=azure_client_id, azure_tenant_id=azure_tenant_id, azure_client_secret="****" if azure_secret_set else "", onedrive_user=onedrive_user, takeoffs_folder=takeoffs_folder, report_to=settings.get("report_to", []), report_cc=settings.get("report_cc", []), ) @app.route("/winston/config", methods=["POST"]) @togen_admin_required def winston_config_post(): """Save Winston configuration (mailbox, ClickUp key, recipients).""" body = request.get_json(silent=True) or {} settings = _load_winston_settings() if "draft_mailbox" in body: val = str(body["draft_mailbox"] or "").strip() settings["draft_mailbox"] = val # Secrets / single-value fields: empty string = leave unchanged. for key in ( "clickup_api_key", "clickup_list_id", "onedrive_user", "takeoffs_folder", "azure_client_id", "azure_client_secret", "azure_tenant_id", ): if key in body: val = str(body[key] or "").strip() if val: settings[key] = val for key in ("report_to", "report_cc"): if key in body: raw = body[key] if not isinstance(raw, list): return jsonify(error=f"{key} must be a list."), 400 cleaned = [] seen = set() for e in raw: if not isinstance(e, str): continue e = e.strip().lower() if e and e not in seen: seen.add(e) cleaned.append(e) settings[key] = cleaned try: _save_winston_settings(settings) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True) @app.route("/winston/schedules") @togen_admin_required def winston_schedules_list(): return jsonify(schedules=_load_winston_schedules()) @app.route("/winston/schedules", methods=["POST"]) @togen_admin_required def winston_schedules_create(): body = request.get_json(silent=True) or {} schedule, err = _winston_schedule_from_body(body) if err: return jsonify(error=err), 400 schedules = _load_winston_schedules() schedules.append(schedule) _save_winston_schedules(schedules) return jsonify(schedule=schedule), 201 @app.route("/winston/schedules/", methods=["PUT"]) @togen_admin_required def winston_schedules_update(sched_id): body = request.get_json(silent=True) or {} schedules = _load_winston_schedules() idx = next((i for i, s in enumerate(schedules) if s.get("id") == sched_id), None) if idx is None: return jsonify(error="Schedule not found."), 404 schedule, err = _winston_schedule_from_body(body, existing=schedules[idx]) if err: return jsonify(error=err), 400 schedules[idx] = schedule _save_winston_schedules(schedules) return jsonify(schedule=schedule) @app.route("/winston/schedules/", methods=["DELETE"]) @togen_admin_required def winston_schedules_delete(sched_id): schedules = _load_winston_schedules() remaining = [s for s in schedules if s.get("id") != sched_id] if len(remaining) == len(schedules): return jsonify(error="Schedule not found."), 404 _save_winston_schedules(remaining) return jsonify(ok=True) @app.route("/winston/date-logic") @togen_admin_required def winston_date_logic_list(): """List Date Logic pipelines + the canonical process order and base rules.""" from togen import winston_date_logic as _wdl return jsonify( pipelines=_wdl.list_pipelines(), base_rules=[{"value": k, "label": v} for k, v in _wdl.BASE_RULES.items()], process_keys=_wdl.PROCESS_KEYS, ) @app.route("/winston/date-logic", methods=["POST"]) @togen_admin_required def winston_date_logic_create(): from togen import winston_date_logic as _wdl body = request.get_json(silent=True) or {} pipeline, err = _wdl.create(body.get("name"), (body.get("base_rule") or "").strip()) if err: return jsonify(error=err), 400 return jsonify(pipeline=pipeline), 201 @app.route("/winston/date-logic/", methods=["PATCH"]) @togen_admin_required def winston_date_logic_update(pipeline_id): from togen import winston_date_logic as _wdl body = request.get_json(silent=True) or {} name = body.get("name") if "name" in body else None process_labels = body.get("process_labels") if "process_labels" in body else None pipeline, err = _wdl.update(pipeline_id, name=name, process_labels=process_labels) if err: status = 404 if err == "Date Logic not found." else 400 return jsonify(error=err), status return jsonify(pipeline=pipeline) @app.route("/winston/date-logic/", methods=["DELETE"]) @togen_admin_required def winston_date_logic_delete(pipeline_id): from togen import winston_date_logic as _wdl referenced = any(s.get("profile") == pipeline_id for s in _load_winston_schedules()) ok, err, status = _wdl.delete(pipeline_id, referenced) if not ok: return jsonify(error=err), status return jsonify(ok=True) @app.route("/winston/run-history") @togen_admin_required def winston_run_history(): def _db_run_to_record(r: dict) -> dict: """Transform bot_diagnostics DB row to the legacy run record shape the UI expects.""" summary: dict = {} if r.get("summary_json"): try: summary = json.loads(r["summary_json"]) except (json.JSONDecodeError, TypeError): pass started = r.get("started_at") or 0 ran_at = ( _datetime_type.fromtimestamp(started, tz=_dt_timezone.utc).isoformat() if started else "" ) return { "id": r["id"], "ran_at": ran_at, "date": r.get("target_date") or "", "triggered_by": r.get("trigger") or "", "status": r.get("status") or "done", "quotes_found": summary.get("quotes_found", 0), "drafts_created": summary.get("drafts_created", 0), "drafts_skipped": summary.get("drafts_skipped", 0), "drafts_failed": summary.get("drafts_failed", 0), "pdf_routed": summary.get("pdf_routed", 0), "pdf_route_failed": summary.get("pdf_route_failed", 0), "error": r.get("error"), "quotes": summary.get("quotes", []), } db_runs = _diag_mod.list_runs("winston", limit=_WINSTON_MAX_RUN_RECORDS) db_ids = {r["id"] for r in db_runs} result = [_db_run_to_record(r) for r in db_runs] # Fallback: include legacy JSON records not yet in bot_diagnostics.db # (one-release read-only migration — retire in next release) for old_rec in _load_winston_run_records(): if old_rec.get("id") not in db_ids: result.append(old_rec) result = result[:_WINSTON_MAX_RUN_RECORDS] return jsonify(records=result) @app.route("/winston/run-now", methods=["POST"]) @togen_admin_required def winston_run_now(): """Kick off the Winston pipeline for a given date in a background thread.""" from togen import winston_pipeline as _wp body = request.get_json(silent=True) or {} date_str = (body.get("date") or "").strip() if not date_str: date_str = _datetime_type.now().strftime("%Y-%m-%d") try: _datetime_type.strptime(date_str, "%Y-%m-%d") except ValueError: return jsonify(error="Invalid date; expected YYYY-MM-DD."), 400 run_id = str(uuid.uuid4()) def _run_bg(rid, dt): try: _wp.run(dt, trigger="user", run_id=rid) except Exception: app.logger.exception("Winston run-now background error run_id=%s", rid) t = threading.Thread(target=_run_bg, args=(run_id, date_str), daemon=True) t.start() return jsonify(run_id=run_id, status="running", date=date_str), 202 # --------------------------------------------------------------------------- # History capture (DVI-789) — shared history.db spine + per-tool adapters. # --------------------------------------------------------------------------- _HISTORY_RETENTION_DAYS = 365 _HISTORY_MAX_ROWS_PER_TOOL = 50_000 def _history_db(): """Open history.db, creating the schema on first use. Caller must close.""" conn = sqlite3.connect(str(HISTORY_DB_PATH), check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.executescript(""" CREATE TABLE IF NOT EXISTS tool_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, tool TEXT NOT NULL, action TEXT NOT NULL, actor TEXT, summary TEXT, ref TEXT, detail TEXT, created_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_hist_tool_time ON tool_history (tool, created_at DESC); """) return conn def _prune_history_db(): """Remove rows older than retention window or exceeding per-tool cap.""" try: conn = _history_db() try: cutoff = ( _datetime_type.utcnow() - _timedelta_type(days=_HISTORY_RETENTION_DAYS) ).strftime("%Y-%m-%dT%H:%M:%SZ") conn.execute( "DELETE FROM tool_history WHERE created_at < ?", (cutoff,) ) tools = [ r[0] for r in conn.execute( "SELECT DISTINCT tool FROM tool_history" ).fetchall() ] for tool in tools: excess = conn.execute( "SELECT id FROM tool_history WHERE tool = ?" " ORDER BY created_at DESC LIMIT -1 OFFSET ?", (tool, _HISTORY_MAX_ROWS_PER_TOOL), ).fetchall() if excess: ids = [r[0] for r in excess] conn.execute( f"DELETE FROM tool_history WHERE id IN ({','.join('?' * len(ids))})", ids, ) conn.commit() finally: conn.close() except Exception: app.logger.exception("history prune error") def record_history(tool, action, *, actor=None, summary="", ref="", detail=None): """Insert one history event. Fire-and-forget; never raises.""" try: import json as _json conn = _history_db() try: created_at = _datetime_type.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") detail_str = _json.dumps(detail) if detail is not None else None conn.execute( "INSERT INTO tool_history (tool, action, actor, summary, ref, detail, created_at)" " VALUES (?, ?, ?, ?, ?, ?, ?)", (tool, action, actor, summary or "", ref or "", detail_str, created_at), ) conn.commit() finally: conn.close() except Exception: pass def _load_history_settings(): """Return {capture: {tool: bool}} — defaults all captured to True.""" defaults = {"capture": {"print": True}} if HISTORY_SETTINGS_FILE.is_file(): try: saved = json.loads(HISTORY_SETTINGS_FILE.read_text()) if isinstance(saved, dict) and isinstance(saved.get("capture"), dict): for k, v in saved["capture"].items(): if isinstance(v, bool): defaults["capture"][k] = v except (json.JSONDecodeError, OSError): pass return defaults def _is_history_capture_enabled(tool): """True when history capture is enabled for this tool (default True).""" return _load_history_settings()["capture"].get(tool, True) # --- Print adapter for /history/print --- def _print_history_as_generic(limit, offset, f_user, f_action): """Adapt print_jobs into the generic history entry shape.""" where, params = [], [] if f_user: where.append("LOWER(j.created_by) LIKE ?") params.append(f"%{f_user}%") if f_action: where.append("LOWER(j.source) LIKE ?") params.append(f"%{f_action}%") clause = (" WHERE " + " AND ".join(where)) if where else "" conn = _print_db() try: total = conn.execute( "SELECT COUNT(*) AS n FROM print_jobs j" + clause, params ).fetchone()["n"] rows = conn.execute( "SELECT j.id, j.created_by, j.source, j.file_ref, j.published_name," " j.kind, j.copies, j.created_at, a.hostname AS machine_name," " c.acked_status" " FROM print_jobs j" " JOIN print_commands c ON c.id = j.id" " LEFT JOIN print_agents a ON a.machine_id = j.machine_id" + clause + " ORDER BY j.created_at DESC LIMIT ? OFFSET ?", params + [limit, offset], ).fetchall() finally: conn.close() entries = [] for r in rows: src = r["source"] or "" label = _PRINT_SOURCE_LABELS.get(src, src or "—") printer = r["published_name"] or r["machine_name"] or "" summary = f"{label} — {r['file_ref'] or r['id']}" if printer: summary += f" → {printer}" entries.append({ "id": r["id"], "tool": "print", "action": src or "print", "actor": r["created_by"] or "", "summary": summary, "ref": r["file_ref"] or "", "created_at": _datetime_type.utcfromtimestamp( float(r["created_at"]) ).strftime("%Y-%m-%dT%H:%M:%SZ") if r["created_at"] else "", "detail": None, }) return entries, total @app.route("/history/") @togen_admin_required def history_tool(tool): """Generic history endpoint. Print is served by an adapter; others from history.db. Query params: limit, offset, user (actor substring), action (action substring), q. """ try: limit = min(int(request.args.get("limit", 100)), 500) offset = max(int(request.args.get("offset", 0)), 0) except (TypeError, ValueError): return jsonify(error="Invalid limit or offset."), 400 f_user = request.args.get("user", "").strip().lower() f_action = request.args.get("action", "").strip().lower() f_q = request.args.get("q", "").strip().lower() if tool == "print": entries, total = _print_history_as_generic(limit, offset, f_user, f_action or f_q) return jsonify(entries=entries, total=total, offset=offset, limit=limit) # Generic history.db path for all other tools. where, params = ["tool = ?"], [tool] if f_user: where.append("LOWER(COALESCE(actor,'')) LIKE ?") params.append(f"%{f_user}%") if f_action: where.append("LOWER(COALESCE(action,'')) LIKE ?") params.append(f"%{f_action}%") if f_q: where.append("(LOWER(COALESCE(summary,'')) LIKE ? OR LOWER(COALESCE(ref,'')) LIKE ?)") params += [f"%{f_q}%", f"%{f_q}%"] clause = " WHERE " + " AND ".join(where) conn = _history_db() try: total = conn.execute( "SELECT COUNT(*) AS n FROM tool_history" + clause, params ).fetchone()["n"] rows = conn.execute( "SELECT id, tool, action, actor, summary, ref, created_at, detail" " FROM tool_history" + clause + " ORDER BY created_at DESC LIMIT ? OFFSET ?", params + [limit, offset], ).fetchall() finally: conn.close() import json as _json entries = [] for r in rows: entries.append({ "id": r["id"], "tool": r["tool"], "action": r["action"], "actor": r["actor"] or "", "summary": r["summary"] or "", "ref": r["ref"] or "", "created_at": r["created_at"], "detail": _json.loads(r["detail"]) if r["detail"] else None, }) return jsonify(entries=entries, total=total, offset=offset, limit=limit) @app.route("/togen-admin/history", methods=["GET"]) @togen_admin_required def togen_admin_history_get(): """Return history capture settings.""" return jsonify(_load_history_settings()) @app.route("/togen-admin/history", methods=["POST"]) @togen_admin_required def togen_admin_history_post(): """Save per-tool history capture toggles.""" body = request.get_json(silent=True) or {} settings = _load_history_settings() if isinstance(body.get("capture"), dict): for k, v in body["capture"].items(): if isinstance(v, bool): settings["capture"][k] = v HISTORY_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) return jsonify(ok=True, capture=settings["capture"]) # --------------------------------------------------------------------------- # Print System (DVI-667 / DVI-657 P1) — agent enrollment, check-in, admin pane. # Mirrors the Vision agent machinery (vision.db tables + /vision/* routes). # --------------------------------------------------------------------------- _PRINT_ENROLL_TTL_S = 15 * 60 _PRINT_OPEN_ENROLL_KEY = "open_enroll_secret" # Authoritative agent status: pending (self-enrolled, quarantined) / active # (quick-enrolled or approved) / revoked (token refused). The legacy `revoked` # INTEGER mirrors status='revoked' for parity with the Vision schema. _PRINT_AGENT_STATUSES = ("pending", "active", "revoked") def _print_db(): """Open the Print SQLite DB, creating tables on first use. Caller must close.""" conn = sqlite3.connect(str(PRINT_DB_PATH), check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.executescript(""" CREATE TABLE IF NOT EXISTS print_enrollment_tokens ( token TEXT PRIMARY KEY, machine_id TEXT, issued_at REAL NOT NULL, expires_at REAL NOT NULL, consumed_at REAL ); CREATE TABLE IF NOT EXISTS print_agents ( machine_id TEXT PRIMARY KEY, hostname TEXT, agent_token TEXT UNIQUE NOT NULL, enrolled_at REAL NOT NULL, last_seen_at REAL, revoked INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'active', os_version TEXT, logged_in_user TEXT, agent_version TEXT ); CREATE TABLE IF NOT EXISTS print_printers ( id INTEGER PRIMARY KEY AUTOINCREMENT, machine_id TEXT NOT NULL, raw_name TEXT NOT NULL, driver TEXT, port TEXT, is_default INTEGER NOT NULL DEFAULT 0, printer_status TEXT, included INTEGER NOT NULL DEFAULT 0, for_documents INTEGER NOT NULL DEFAULT 0, for_labels INTEGER NOT NULL DEFAULT 0, published_name TEXT, first_seen_at REAL, last_seen_at REAL, UNIQUE (machine_id, raw_name), FOREIGN KEY (machine_id) REFERENCES print_agents (machine_id) ); CREATE TABLE IF NOT EXISTS print_kv ( key TEXT PRIMARY KEY, value TEXT, updated_at REAL ); CREATE TABLE IF NOT EXISTS print_commands ( id TEXT PRIMARY KEY, machine_id TEXT NOT NULL, action TEXT NOT NULL, params_json TEXT, created_at REAL NOT NULL, delivered_at REAL, acked_at REAL, acked_status TEXT, error_message TEXT, FOREIGN KEY (machine_id) REFERENCES print_agents (machine_id) ); CREATE TABLE IF NOT EXISTS print_jobs ( id TEXT PRIMARY KEY, created_by TEXT, source TEXT, file_ref TEXT, file_path TEXT NOT NULL, machine_id TEXT NOT NULL, printer_raw TEXT NOT NULL, published_name TEXT, kind TEXT NOT NULL, copies INTEGER NOT NULL DEFAULT 1, created_at REAL NOT NULL, FOREIGN KEY (id) REFERENCES print_commands (id), FOREIGN KEY (machine_id) REFERENCES print_agents (machine_id) ); CREATE INDEX IF NOT EXISTS idx_print_printers_machine ON print_printers (machine_id); CREATE INDEX IF NOT EXISTS idx_print_jobs_creator ON print_jobs (created_by); """) return conn def _print_kv_get(conn, key): row = conn.execute("SELECT value FROM print_kv WHERE key = ?", (key,)).fetchone() return row["value"] if row else None def _print_kv_set(conn, key, value): conn.execute( "INSERT INTO print_kv (key, value, updated_at) VALUES (?, ?, ?)" " ON CONFLICT(key) DO UPDATE SET value = excluded.value," " updated_at = excluded.updated_at", (key, value, time.time()), ) def _print_get_open_enroll_secret(conn, create=True): """Return the current open-enroll shared secret, minting one on first use.""" secret = _print_kv_get(conn, _PRINT_OPEN_ENROLL_KEY) if not secret and create: secret = uuid.uuid4().hex _print_kv_set(conn, _PRINT_OPEN_ENROLL_KEY, secret) conn.commit() return secret def _print_upsert_printers(conn, machine_id, printers): """Upsert agent-reported printers. Raw fields (driver/port/is_default/status) refresh on every check-in; admin-set config (included / for_documents / for_labels / published_name) is preserved across reports.""" now = time.time() for p in printers or []: if not isinstance(p, dict): continue raw_name = (p.get("raw_name") or "").strip() if not raw_name: continue conn.execute( """INSERT INTO print_printers (machine_id, raw_name, driver, port, is_default, printer_status, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(machine_id, raw_name) DO UPDATE SET driver = excluded.driver, port = excluded.port, is_default = excluded.is_default, printer_status = excluded.printer_status, last_seen_at = excluded.last_seen_at""", (machine_id, raw_name, p.get("driver") or "", p.get("port") or "", 1 if p.get("is_default") else 0, p.get("status") or "", now, now), ) @app.route("/print/agents/enroll", methods=["POST"]) @togen_admin_required def print_agents_enroll(): """Issue a short-lived enrollment token. machine_id optional: supplying it pre-binds the token (quick-enroll → active); omitting it lands the agent pending (admin approval required).""" body = request.get_json(silent=True) or {} machine_id = (body.get("machine_id") or "").strip() or None token = str(uuid.uuid4()) now = time.time() conn = _print_db() try: conn.execute( "INSERT INTO print_enrollment_tokens (token, machine_id, issued_at, expires_at)" " VALUES (?, ?, ?, ?)", (token, machine_id, now, now + _PRINT_ENROLL_TTL_S), ) conn.commit() finally: conn.close() return jsonify(enrollment_token=token, machine_id=machine_id, mode="quick" if machine_id else "self") @app.route("/print/agents/register", methods=["POST"]) def print_agent_register(): """Machine presents a credential; receives a permanent agent_token. Three trust paths (mirrors Vision): machine-bound token → active; NULL-machine token → pending; open-enroll secret → pending. """ body = request.get_json(silent=True) or {} auth = request.headers.get("Authorization", "") credential = (body.get("enrollment_token") or (auth[7:].strip() if auth.startswith("Bearer ") else "")) open_secret_body = (body.get("open_enroll_secret") or "").strip() machine_id_body = (body.get("machine_id") or "").strip() hostname = (body.get("hostname") or "").strip() or None if not credential and not open_secret_body: return jsonify(error="enrollment_token or open_enroll_secret required."), 400 now = time.time() conn = _print_db() try: open_secret = _print_get_open_enroll_secret(conn, create=False) via_open = bool(open_secret) and ( open_secret_body == open_secret or credential == open_secret ) if via_open: if not machine_id_body: return jsonify(error="machine_id required for self-enroll."), 400 effective_machine_id, status = machine_id_body, "pending" else: row = conn.execute( "SELECT * FROM print_enrollment_tokens WHERE token = ?", (credential,) ).fetchone() if not row: return jsonify(error="Invalid enrollment_token."), 401 if row["consumed_at"] is not None: return jsonify(error="enrollment_token already used."), 401 if now > row["expires_at"]: return jsonify(error="enrollment_token expired."), 401 if row["machine_id"]: if machine_id_body and machine_id_body != row["machine_id"]: return jsonify(error="machine_id does not match enrollment."), 403 effective_machine_id, status = row["machine_id"], "active" else: if not machine_id_body: return jsonify(error="machine_id required for self-enroll."), 400 effective_machine_id, status = machine_id_body, "pending" conn.execute( "UPDATE print_enrollment_tokens SET consumed_at = ? WHERE token = ?", (now, credential), ) existing = conn.execute( "SELECT agent_token, status FROM print_agents WHERE machine_id = ?", (effective_machine_id,), ).fetchone() if existing: agent_token = existing["agent_token"] status = existing["status"] # re-register keeps prior disposition else: agent_token = str(uuid.uuid4()) conn.execute( "INSERT INTO print_agents" " (machine_id, hostname, agent_token, enrolled_at, status, revoked)" " VALUES (?, ?, ?, ?, ?, ?)", (effective_machine_id, hostname, agent_token, now, status, 1 if status == "revoked" else 0), ) conn.commit() finally: conn.close() return jsonify(agent_token=agent_token, machine_id=effective_machine_id, status=status) @app.route("/print/checkin", methods=["POST"]) def print_checkin(): """Agent → server check-in. Validates the bearer agent_token, refreshes the agent fingerprint, upserts reported printers, and dequeues the oldest unacked command (acking the prior delivery).""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 conn = _print_db() try: agent = conn.execute( "SELECT machine_id, status FROM print_agents WHERE agent_token = ? AND revoked = 0", (token,), ).fetchone() if not agent: return jsonify(error="Invalid or revoked agent_token."), 401 machine_id = agent["machine_id"] agent_status = agent["status"] body = request.get_json(silent=True) or {} now = time.time() conn.execute( "UPDATE print_agents SET last_seen_at = ?," " hostname = COALESCE(?, hostname)," " os_version = COALESCE(?, os_version)," " logged_in_user = COALESCE(?, logged_in_user)," " agent_version = COALESCE(?, agent_version)" " WHERE machine_id = ?", (now, body.get("hostname"), body.get("os_version"), body.get("logged_in_user"), body.get("agent_version"), machine_id), ) # Only trust reported printers from an approved agent. if agent_status == "active": _print_upsert_printers(conn, machine_id, body.get("printers")) cmd_row = conn.execute( "SELECT id, action, params_json FROM print_commands" " WHERE machine_id = ? AND acked_at IS NULL" " ORDER BY created_at ASC LIMIT 1", (machine_id,), ).fetchone() if cmd_row: conn.execute( "UPDATE print_commands SET delivered_at = COALESCE(delivered_at, ?)" " WHERE id = ?", (now, cmd_row["id"]), ) conn.commit() finally: conn.close() command = None if cmd_row: params = None if cmd_row["params_json"]: try: params = json.loads(cmd_row["params_json"]) except (json.JSONDecodeError, TypeError): params = None command = {"id": cmd_row["id"], "action": cmd_row["action"], "params": params} return jsonify(ack=True, next_interval_s=10, command=command, agent_status=agent_status) @app.route("/print/commands//ack", methods=["POST"]) def print_command_ack(cmd_id): """Agent acknowledges execution of a delivered command.""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 body = request.get_json(silent=True) or {} status = (body.get("status") or "").strip() if status not in ("ok", "error"): return jsonify(error="status must be 'ok' or 'error'."), 400 error_message = body.get("error_message") or None now = time.time() conn = _print_db() try: agent = conn.execute( "SELECT machine_id FROM print_agents WHERE agent_token = ? AND revoked = 0", (token,), ).fetchone() if not agent: return jsonify(error="Invalid or revoked agent_token."), 401 cmd = conn.execute( "SELECT id FROM print_commands WHERE id = ? AND machine_id = ?", (cmd_id, agent["machine_id"]), ).fetchone() if not cmd: return jsonify(error="Command not found."), 404 conn.execute( "UPDATE print_commands SET acked_at = ?, acked_status = ?, error_message = ?" " WHERE id = ?", (now, status, error_message, cmd_id), ) conn.commit() finally: conn.close() return jsonify(ack=True) @app.route("/print/agents") @togen_admin_required def print_agents_list(): """List enrolled print agents with last-seen + status. Pending agents carry their fingerprint so the Print pane can render an approval card.""" now = time.time() conn = _print_db() try: rows = conn.execute( "SELECT machine_id, hostname, enrolled_at, last_seen_at, revoked, status," " os_version, logged_in_user, agent_version" " FROM print_agents ORDER BY enrolled_at DESC" ).fetchall() finally: conn.close() agents = [] for r in rows: last_seen = r["last_seen_at"] agents.append({ "machine_id": r["machine_id"], "hostname": r["hostname"], "enrolled_at": r["enrolled_at"], "last_seen": int(last_seen) if last_seen else None, "online": bool(last_seen) and (now - last_seen) <= PRINT_STALE_SECONDS, "status": r["status"], "revoked": bool(r["revoked"]), "fingerprint": { "hostname": r["hostname"], "os_version": r["os_version"], "logged_in_user": r["logged_in_user"], "agent_version": r["agent_version"], }, }) return jsonify(agents=agents) def _print_set_agent_status(machine_id, new_status, require_pending=False): """Shared approve/deny/revoke transition. Returns (payload, http_status).""" machine_id = (machine_id or "").strip() conn = _print_db() try: row = conn.execute( "SELECT status FROM print_agents WHERE machine_id = ?", (machine_id,) ).fetchone() if not row: return {"error": "machine_id not found."}, 404 if require_pending and row["status"] != "pending": return {"error": f"machine is '{row['status']}', not pending."}, 409 conn.execute( "UPDATE print_agents SET status = ?, revoked = ? WHERE machine_id = ?", (new_status, 1 if new_status == "revoked" else 0, machine_id), ) conn.commit() finally: conn.close() return {"ok": True, "machine_id": machine_id, "status": new_status}, 200 @app.route("/print/agents//approve", methods=["POST"]) @togen_admin_required def print_agent_approve(machine_id): """Approve a self-enrolled print agent (pending → active).""" payload, code = _print_set_agent_status(machine_id, "active", require_pending=True) return jsonify(**payload), code @app.route("/print/agents//deny", methods=["POST"]) @togen_admin_required def print_agent_deny(machine_id): """Deny a self-enrolled print agent (pending → revoked).""" payload, code = _print_set_agent_status(machine_id, "revoked", require_pending=True) return jsonify(**payload), code @app.route("/print/agents//revoke", methods=["POST"]) @togen_admin_required def print_agent_revoke(machine_id): """Revoke a print agent's token (status=revoked); future check-ins refused.""" payload, code = _print_set_agent_status(machine_id, "revoked") return jsonify(**payload), code @app.route("/print/agents/", methods=["DELETE"]) @togen_admin_required def print_agent_delete(machine_id): """Delete a revoked print agent and its orphan rows (printers/commands/jobs). Only revoked agents are removable (409 otherwise) so a live agent can't be silently dropped out from under an active check-in loop.""" machine_id = (machine_id or "").strip() conn = _print_db() try: row = conn.execute( "SELECT revoked FROM print_agents WHERE machine_id = ?", (machine_id,) ).fetchone() if not row: return jsonify(error="machine_id not found."), 404 if not row["revoked"]: return jsonify(error="agent is not revoked; revoke it before deleting."), 409 conn.execute("DELETE FROM print_jobs WHERE machine_id = ?", (machine_id,)) conn.execute("DELETE FROM print_commands WHERE machine_id = ?", (machine_id,)) conn.execute("DELETE FROM print_printers WHERE machine_id = ?", (machine_id,)) conn.execute("DELETE FROM print_agents WHERE machine_id = ?", (machine_id,)) conn.commit() finally: conn.close() return jsonify(ok=True, machine_id=machine_id, deleted=True) @app.route("/print/agents//printers") @togen_admin_required def print_agent_printers_get(machine_id): """List the printers discovered by an agent with their Togen config.""" machine_id = (machine_id or "").strip() conn = _print_db() try: agent = conn.execute( "SELECT machine_id, hostname, status FROM print_agents WHERE machine_id = ?", (machine_id,), ).fetchone() if not agent: return jsonify(error="machine_id not found."), 404 rows = conn.execute( "SELECT id, raw_name, driver, port, is_default, printer_status," " included, for_documents, for_labels, published_name, last_seen_at" " FROM print_printers WHERE machine_id = ? ORDER BY raw_name COLLATE NOCASE", (machine_id,), ).fetchall() finally: conn.close() printers = [{ "id": r["id"], "raw_name": r["raw_name"], "driver": r["driver"], "port": r["port"], "is_default": bool(r["is_default"]), "printer_status": r["printer_status"], "included": bool(r["included"]), "for_documents": bool(r["for_documents"]), "for_labels": bool(r["for_labels"]), "published_name": r["published_name"] or "", "last_seen": int(r["last_seen_at"]) if r["last_seen_at"] else None, } for r in rows] return jsonify(machine_id=machine_id, hostname=agent["hostname"], status=agent["status"], printers=printers) @app.route("/print/agents//printers", methods=["POST"]) @togen_admin_required def print_agent_printers_post(machine_id): """Save per-printer Togen config: included / for_documents / for_labels / published_name. Body: {printers: [{id, included, for_documents, for_labels, published_name}, ...]}. Only printers belonging to this agent are updated.""" machine_id = (machine_id or "").strip() body = request.get_json(silent=True) or {} items = body.get("printers") if not isinstance(items, list): return jsonify(error="printers list required."), 400 conn = _print_db() try: agent = conn.execute( "SELECT machine_id FROM print_agents WHERE machine_id = ?", (machine_id,) ).fetchone() if not agent: return jsonify(error="machine_id not found."), 404 for it in items: if not isinstance(it, dict) or "id" not in it: continue try: pid = int(it["id"]) except (TypeError, ValueError): continue included = 1 if it.get("included") else 0 for_documents = 1 if it.get("for_documents") else 0 for_labels = 1 if it.get("for_labels") else 0 published = str(it.get("published_name") or "").strip() conn.execute( "UPDATE print_printers SET included = ?, for_documents = ?," " for_labels = ?, published_name = ? WHERE id = ? AND machine_id = ?", (included, for_documents, for_labels, published, pid, machine_id), ) conn.commit() finally: conn.close() return jsonify(ok=True) @app.route("/print/enroll-secret") @togen_admin_required def print_enroll_secret_get(): """Return the current open-enroll shared secret (minted on first read).""" conn = _print_db() try: secret = _print_get_open_enroll_secret(conn, create=True) finally: conn.close() return jsonify(open_enroll_secret=secret) @app.route("/print/enroll-secret/rotate", methods=["POST"]) @togen_admin_required def print_enroll_secret_rotate(): """Rotate the open-enroll secret. Prior secret stops working immediately; already-enrolled agents keep their agent_token (unaffected).""" new_secret = uuid.uuid4().hex conn = _print_db() try: _print_kv_set(conn, _PRINT_OPEN_ENROLL_KEY, new_secret) conn.commit() finally: conn.close() return jsonify(open_enroll_secret=new_secret) # --------------------------------------------------------------------------- # Print job dispatch (DVI-669 / DVI-657 P3) # --------------------------------------------------------------------------- # Server-side spool for PDFs produced by job creation (e.g. image→PDF wraps). # Originals (already-PDF refs) are referenced in place; only conversions land here. PRINT_SPOOL_DIR = Path( os.environ.get("PRINT_SPOOL_DIR") or (Path(tempfile.gettempdir()) / "togen_print_spool") ) # file_ref sources mapped to the directory root they resolve against. Anything # else (and the "wor" pseudo-source) is handled explicitly in the resolver. _PRINT_SOURCE_ROOTS = { "watch": WATCH_DIR, "projectbuilder": PROJECTBUILDER_DIR, "shared": SHARED_DIR, # File Browser + Single Carrier Sheet (DVI-670) "mtslabels": MTS_LABELS_DIR, # MTS Label Creator (DVI-670) } _PRINT_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tif", ".tiff"} _PRINT_MAX_COPIES = 50 def _print_roots_allowlist(): """Absolute roots a resolved print file may live under (defense in depth).""" roots = [WATCH_DIR.resolve(), PROJECTBUILDER_DIR.resolve(), PRINT_SPOOL_DIR.resolve(), SHARED_DIR.resolve(), MTS_LABELS_DIR.resolve(), PRODRUN_DIR.resolve()] try: roots.extend(d.resolve() for d in _wor_type_dirs(None)) except Exception: pass return roots def _print_path_within_allowlist(path): p = str(Path(path).resolve()) for root in _print_roots_allowlist(): r = str(root) if p == r or p.startswith(r + os.sep): return True return False def _print_ensure_pdf(path): """Return a printable PDF path for ``path``. PDFs pass through; raster images are wrapped into a one-page PDF in the spool dir. Returns (pdf_path, error).""" src = Path(path) ext = src.suffix.lower() if ext == ".pdf": return src, None if ext in _PRINT_IMAGE_EXTS: try: PRINT_SPOOL_DIR.mkdir(parents=True, exist_ok=True) out = PRINT_SPOOL_DIR / f"{uuid.uuid4().hex}.pdf" with _pil_image.open(src) as im: im.convert("RGB").save(str(out), "PDF", resolution=300.0) return out, None except Exception as exc: # noqa: BLE001 — surface conversion failure to caller return None, f"Could not convert image to PDF: {exc}" return None, f"Unsupported file type for printing: {ext or '(none)'}" def _resolve_print_file(source, file_ref): """Resolve a (source, file_ref) pair to a server-side PDF path. Sources are allowlisted directory roots; ``file_ref`` is always treated as a path relative to that root and is traversal-guarded. The ``wor`` pseudo-source delegates to the WOR file finder. Non-PDF raster images are converted to PDF. Returns (pdf_path, error_message).""" source = (source or "").strip().lower() file_ref = (file_ref or "").strip() if not file_ref: return None, "file_ref is required." if source == "wor": resolved = _resolve_wor_file(file_ref) if not resolved: return None, "WOR file not found." return _print_ensure_pdf(resolved) if source == "prodrun": # file_ref is "/"; delegate to the merge-file finder # (it searches the job's date subdirs and is traversal-guarded). job_id, _, fname = file_ref.partition("/") if not job_id or not fname or ".." in job_id or ".." in fname: return None, "Invalid file_ref." resolved = _prodrun_find_file(job_id, fname) if not resolved: return None, "File not found." return _print_ensure_pdf(resolved) root = _PRINT_SOURCE_ROOTS.get(source) if root is None: return None, f"Unknown print source: {source or '(none)'}" if ".." in file_ref: return None, "Invalid file_ref." root_resolved = root.resolve() candidate = (root_resolved / file_ref.lstrip("/")).resolve() if candidate != root_resolved and not str(candidate).startswith(str(root_resolved) + os.sep): return None, "file_ref escapes the source root." if not candidate.is_file(): return None, "File not found." return _print_ensure_pdf(candidate) def _resolve_published_printer(published_name): """Map a published printer name to (machine_id, raw_name) for an included printer on an active agent. Falls back to matching raw_name. Prefers the most-recently-seen printer when several match. Returns None if unresolved.""" name = (published_name or "").strip() if not name: return None conn = _print_db() try: rows = conn.execute( "SELECT p.machine_id, p.raw_name, p.published_name, p.last_seen_at" " FROM print_printers p" " JOIN print_agents a ON a.machine_id = p.machine_id" " WHERE p.included = 1 AND a.status = 'active'" " ORDER BY p.last_seen_at DESC" ).fetchall() finally: conn.close() low = name.lower() # Prefer an explicit published_name match, then raw_name. for r in rows: if (r["published_name"] or "").strip().lower() == low: return r["machine_id"], r["raw_name"] for r in rows: if (r["raw_name"] or "").strip().lower() == low: return r["machine_id"], r["raw_name"] return None def _print_kind_settings(kind): """SumatraPDF -print-settings string per document kind. Documents shrink to the printable area; labels print at native size (no scaling) so barcodes and QR modules keep their exact dimensions.""" return "noscale" if kind == "label" else "shrink" def _print_job_status(cmd_row): """Derive a job status from its command row's lifecycle timestamps.""" if cmd_row["acked_at"] is not None: return "done" if cmd_row["acked_status"] == "ok" else "error" if cmd_row["delivered_at"] is not None: return "printing" return "queued" @app.route("/print/options") @login_required def print_options(): """Printer choices + the caller's effective defaults for the print popups. Returns ``{available_printers: {documents, labels}, effective_defaults: {document_printer, label_printer}}``. The tool Print buttons (DVI-670 / P4) use this to pre-select the effective default and populate the dropdown.""" email = session["user"].get("email", "") return jsonify( available_printers=_available_printers(), effective_defaults=_effective_default_printers(email), ) @app.route("/print/jobs", methods=["POST"]) @login_required def print_jobs_create(): """Create a print job: resolve the file + target printer, then enqueue a ``print`` command for the owning agent. (DVI-669 / DVI-657 P3) Body: {source, file_ref, printer|"__default__"|"", kind: document|label, copies?} """ if not _user_can_print(session.get("user")): return jsonify(error="The Print system is currently disabled."), 403 email = session["user"].get("email", "") body = request.get_json(silent=True) or {} kind = (body.get("kind") or "document").strip().lower() if kind not in ("document", "label"): return jsonify(error="kind must be 'document' or 'label'."), 400 try: copies = int(body.get("copies") or 1) except (TypeError, ValueError): return jsonify(error="copies must be an integer."), 400 if copies < 1 or copies > _PRINT_MAX_COPIES: return jsonify(error=f"copies must be between 1 and {_PRINT_MAX_COPIES}."), 400 # Resolve the target printer's published name (explicit, or the user's # effective default for this kind via the P2 resolver). requested = body.get("printer") if requested in (None, "", PRINT_USER_DEFAULT): defaults = _effective_default_printers(email) published = defaults.get("label_printer" if kind == "label" else "document_printer", "") if not published: return jsonify(error=f"No default {kind} printer configured."), 400 else: published = str(requested).strip() resolved_printer = _resolve_published_printer(published) if not resolved_printer: return jsonify(error=f"Printer not available: {published}"), 400 machine_id, raw_name = resolved_printer pdf_path, ferr = _resolve_print_file(body.get("source"), body.get("file_ref")) if ferr: # Missing file → 404; everything else is a bad request. code = 404 if ferr in ("File not found.", "WOR file not found.") else 400 return jsonify(error=ferr), code job_id = str(uuid.uuid4()) now = time.time() params = { "printer": raw_name, "kind": kind, "copies": copies, "options": {"print_settings": _print_kind_settings(kind)}, } conn = _print_db() try: conn.execute( "INSERT INTO print_commands (id, machine_id, action, params_json, created_at)" " VALUES (?, ?, 'print', ?, ?)", (job_id, machine_id, json.dumps(params), now), ) conn.execute( "INSERT INTO print_jobs (id, created_by, source, file_ref, file_path," " machine_id, printer_raw, published_name, kind, copies, created_at)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (job_id, email, (body.get("source") or "").strip(), (body.get("file_ref") or "").strip(), str(pdf_path), machine_id, raw_name, published, kind, copies, now), ) conn.commit() finally: conn.close() return jsonify(job_id=job_id, status="queued", printer=published, machine_id=machine_id, kind=kind, copies=copies), 201 @app.route("/print/jobs/") @login_required def print_job_status(job_id): """Return a print job's status. Visible to its creator or any Togen admin.""" email = session["user"].get("email", "") conn = _print_db() try: job = conn.execute( "SELECT j.id, j.created_by, j.published_name, j.kind, j.copies," " j.created_at, c.delivered_at, c.acked_at, c.acked_status," " c.error_message" " FROM print_jobs j JOIN print_commands c ON c.id = j.id" " WHERE j.id = ?", (job_id,), ).fetchone() finally: conn.close() if not job: return jsonify(error="Job not found."), 404 if job["created_by"] != email and not session["user"].get("togen_admin"): return jsonify(error="Forbidden."), 403 return jsonify( job_id=job["id"], status=_print_job_status(job), printer=job["published_name"] or "", kind=job["kind"], copies=job["copies"], created_at=job["created_at"], error_message=job["error_message"] or None, ) @app.route("/print/jobs//file") def print_job_file(job_id): """Agent download of a job's PDF. Gated by the agent bearer token; the job must belong to the requesting agent's machine.""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 conn = _print_db() try: agent = conn.execute( "SELECT machine_id FROM print_agents WHERE agent_token = ? AND revoked = 0", (token,), ).fetchone() if not agent: return jsonify(error="Invalid or revoked agent_token."), 401 job = conn.execute( "SELECT machine_id, file_path FROM print_jobs WHERE id = ?", (job_id,) ).fetchone() finally: conn.close() if not job: return jsonify(error="Job not found."), 404 if job["machine_id"] != agent["machine_id"]: return jsonify(error="Forbidden."), 403 file_path = job["file_path"] if not _print_path_within_allowlist(file_path) or not os.path.isfile(file_path): return jsonify(error="Job file unavailable."), 404 return send_file(file_path, mimetype="application/pdf", as_attachment=True, download_name=f"{job_id}.pdf") @app.route("/enroll/print-agent.zip") def print_enroll_agent_zip(): """Serve the frozen print-agent bundle (exe + installer). 404 with guidance until the artifact is built and deployed. No auth: the ZIP has no secrets.""" path = PRINT_AGENT_DIST_DIR / PRINT_AGENT_ZIP_NAME if not path.is_file(): return jsonify( error="Print agent bundle not available yet.", detail=f"Expected {PRINT_AGENT_ZIP_NAME} in PRINT_AGENT_DIST_DIR " f"({PRINT_AGENT_DIST_DIR}); run build_print_agent.ps1 on a " f"Windows host to produce the bundle, then deploy to the server.", ), 404 return send_file(str(path), as_attachment=True, download_name=PRINT_AGENT_ZIP_NAME) @app.route("/enroll/print-bootstrap.ps1") def print_enroll_bootstrap(): """Render a PowerShell onboarding script that downloads the Print Agent bundle and enrolls it. ?token= embeds a per-machine token (quick-enroll → active); otherwise the current open-enroll secret is embedded (self-enroll → pending). Mirrors /enroll/bootstrap.ps1 but targets the Print bundle and install-print.ps1. Auth: the script runs on a fresh machine with no Togen session cookie, so the quick-enroll path authenticates via the ?token= enrollment token itself (the same bearer credential /print/agents/register already trusts). The token-less self-enroll path embeds the open-enroll secret, so it stays gated behind a Togen Admin session to avoid serving that secret anonymously. Requiring an admin session unconditionally would 302-redirect the sessionless machine to the Microsoft SSO page, whose HTML then gets piped into `iex` (DVI-674).""" base = _vision_server_base_url() token = (request.args.get("token") or "").strip() if token: credential, mode = token, "quick" else: if not (session.get("user") and session["user"].get("togen_admin")): return ( "Write-Host 'ERROR: This bootstrap URL needs a ?token= enrollment " "token (generate one via Togen Admin -> Agents -> Print -> Enroll), " "or download it from an authenticated Togen Admin session.' " "-ForegroundColor Red\nexit 1\n", 403, {"Content-Type": "text/plain; charset=utf-8", "Cache-Control": "no-store"}, ) conn = _print_db() try: credential = _print_get_open_enroll_secret(conn, create=True) finally: conn.close() mode = "self" zip_url = f"{base}/enroll/print-agent.zip" ps_cred = credential.replace("'", "''") ps_base = base.replace("'", "''") script = f"""# Togen Print agent bootstrap ({mode}-enroll) — generated {time.strftime('%Y-%m-%d %H:%M:%S')} # Run as Administrator. Downloads the agent bundle and delegates to install-print.ps1. $ErrorActionPreference = 'Stop' $Server = '{ps_base}' $Credential = '{ps_cred}' $Zip = Join-Path $env:TEMP 'togen-print-agent.zip' $ExtractDir = Join-Path $env:TEMP 'togen-print-agent' # -- Download (no auth required; ZIP contains no secrets) ------------------- Write-Host 'Downloading Togen Print agent...' try {{ Invoke-WebRequest -Uri '{zip_url}' -OutFile $Zip -UseBasicParsing }} catch {{ Write-Host "ERROR: Download failed: $_" -ForegroundColor Red Write-Host 'Ensure the print agent bundle has been built and deployed. Contact your IT admin.' exit 1 }} $ZipSize = (Get-Item $Zip -ErrorAction SilentlyContinue).Length if (-not $ZipSize -or $ZipSize -lt 4096) {{ Write-Host 'ERROR: Print agent bundle not available on this server yet.' -ForegroundColor Red Write-Host 'The Windows build step must be run first. Contact your IT admin.' Remove-Item $Zip -Force -ErrorAction SilentlyContinue exit 1 }} # -- Extract ---------------------------------------------------------------- if (Test-Path $ExtractDir) {{ Remove-Item -Recurse -Force $ExtractDir }} Expand-Archive -Path $Zip -DestinationPath $ExtractDir -Force Remove-Item $Zip -Force -ErrorAction SilentlyContinue # -- Install (enroll + Scheduled Task via install-print.ps1 from the bundle) - $Installer = Join-Path $ExtractDir 'install-print.ps1' if (-not (Test-Path $Installer)) {{ Write-Host 'ERROR: install-print.ps1 missing from agent bundle.' -ForegroundColor Red exit 1 }} Write-Host 'Running installer...' & powershell.exe -ExecutionPolicy Bypass -NonInteractive -File $Installer -Server $Server -Enroll $Credential Write-Host '' Write-Host 'Bootstrap complete.' -ForegroundColor Green Write-Host 'If this machine registered as PENDING, visit Togen Admin' Write-Host ' -> Agents -> Print -> Status tab -> Approve this machine.' """ return script, 200, { "Content-Type": "text/plain; charset=utf-8", "Content-Disposition": "attachment; filename=install-print.ps1", "Cache-Control": "no-store", } # --------------------------------------------------------------------------- # Routes — Assistant (DVI-529) # --------------------------------------------------------------------------- @app.route("/togen-admin/assistant") @togen_admin_required def togen_admin_assistant_get(): """Return global assistant settings (key masked). Admin only.""" from assistant import load_assistant_settings, mask_key, ASSISTANT_PROVIDERS cfg = load_assistant_settings() return jsonify( enabled=cfg.get("enabled", False), accessible_to_users=cfg.get("accessible_to_users", False), default_context_md=cfg.get("default_context_md", ""), api_key=mask_key(cfg.get("api_key")), providers=list(ASSISTANT_PROVIDERS.values()), ) @app.route("/togen-admin/assistant", methods=["POST"]) @togen_admin_required def togen_admin_assistant_post(): """Update global assistant settings. Admin only. Accepts: enabled, accessible_to_users, default_context_md, api_key. Key is write-only; omit the field to leave existing key unchanged. """ from assistant import load_assistant_settings, save_assistant_settings body = request.get_json(silent=True) or {} cfg = load_assistant_settings() if "enabled" in body: cfg["enabled"] = bool(body["enabled"]) if "accessible_to_users" in body: cfg["accessible_to_users"] = bool(body["accessible_to_users"]) if "default_context_md" in body: cfg["default_context_md"] = str(body["default_context_md"]) if "api_key" in body: key_val = str(body["api_key"]).strip() if body["api_key"] else "" if key_val and not all(c in "•" for c in key_val): cfg["api_key"] = key_val elif key_val == "" and "api_key" in cfg: del cfg["api_key"] save_assistant_settings(cfg) return jsonify(ok=True) @app.route("/assistant/connection") @login_required def assistant_connection_get(): """Return the current user's assistant connection (key masked).""" from assistant import get_user_assistant, mask_key, ASSISTANT_PROVIDERS email = session["user"]["email"] user_cfg = _load_user_settings(email) ua = get_user_assistant(user_cfg) return jsonify( api_key=mask_key(ua.get("api_key")), model=ua.get("model", ""), providers=list(ASSISTANT_PROVIDERS.values()), ) @app.route("/assistant/status") @login_required def assistant_status(): """Live status for the topbar Assistant indicator dot (DVI-629). available -> MCP/provider connection usable (drives green vs red) busy -> a background prompt/workflow is running (drives yellow). No background processes exist yet, so this is always False today; the field is here so the future feature flips it without a frontend change. """ from assistant import is_assistant_connected email = session["user"]["email"] is_admin = session["user"].get("togen_admin", False) user_cfg = _load_user_settings(email) available = is_assistant_connected(email, is_admin, user_cfg) return jsonify(available=available, busy=False) @app.route("/assistant/connection", methods=["POST"]) @login_required def assistant_connection_post(): """Save the current user's assistant connection (provider + key + model). Key is write-only; omit or send empty to clear. """ from assistant import get_user_assistant, set_user_assistant email = session["user"]["email"] body = request.get_json(silent=True) or {} user_cfg = _load_user_settings(email) ua = dict(get_user_assistant(user_cfg)) if "model" in body: ua["model"] = str(body["model"]).strip() if "api_key" in body: key_val = str(body["api_key"]).strip() if body["api_key"] else "" if key_val and not all(c in "•" for c in key_val): ua["api_key"] = key_val elif key_val == "": ua.pop("api_key", None) user_cfg = set_user_assistant(user_cfg, ua) _save_user_settings(email, user_cfg) return jsonify(ok=True) @app.route("/assistant/context") @login_required def assistant_context_get(): """Return the effective context markdown for the current user.""" from assistant import _effective_context_md email = session["user"]["email"] user_cfg = _load_user_settings(email) return jsonify(context_md=_effective_context_md(user_cfg)) @app.route("/assistant/context", methods=["PUT"]) @login_required def assistant_context_put(): """Set the current user's context md override.""" from assistant import get_user_assistant, set_user_assistant email = session["user"]["email"] body = request.get_json(silent=True) or {} user_cfg = _load_user_settings(email) ua = dict(get_user_assistant(user_cfg)) ua["context_md"] = str(body.get("context_md", "")) user_cfg = set_user_assistant(user_cfg, ua) _save_user_settings(email, user_cfg) return jsonify(ok=True) @app.route("/assistant/chat", methods=["POST"]) @login_required def assistant_chat(): """Proxy a chat request to the configured LLM provider. Request body: { messages: [{role, content}], context_md?: str } context_md overrides the effective context for this request (optional). Returns: { reply: str, agentic: bool } OR { pending_action: {token, tool, display_name, display_args}, partial_reply: str, agentic: bool } when a Tier-1 (mutating) tool needs user confirmation. Agentic mode (DVI-557/560/561): any user with at least one authorized tool gets tool-calling via chat_agentic() — each user sees only the tools their role permits. Users with no tool access fall back to plain chat(). """ from assistant import ( is_assistant_connected, chat, chat_agentic, has_any_tool_access, AssistantUnavailable, AssistantError, PendingActionRequired, _effective_context_md, ) user = session["user"] email = user["email"] is_admin = bool(user.get("togen_admin")) user_cfg = _load_user_settings(email) if not is_assistant_connected(email, is_admin, user_cfg): return jsonify(error="Assistant is not available. Configure an API key in Settings."), 403 body = request.get_json(silent=True) or {} messages = body.get("messages") or [] if not isinstance(messages, list) or not messages: return jsonify(error="messages must be a non-empty list."), 400 ctx_md = body.get("context_md") if ctx_md is None: ctx_md = _effective_context_md(user_cfg) system = ctx_md.strip() if ctx_md else "" # Phase C (DVI-561): any user with at least one authorized tool uses the # agentic loop; chat_agentic() filters tools to the user's own roles. is_agentic = is_admin or has_any_tool_access(user) try: if is_agentic: reply = chat_agentic(messages, system, user=user, user_settings=user_cfg) else: reply = chat(messages, system, user_settings=user_cfg) except PendingActionRequired as exc: return jsonify( pending_action={ "token": exc.token, "tool": exc.tool, "display_name": exc.display_name, "display_args": exc.display_args, }, partial_reply=exc.partial_reply, agentic=True, ) except AssistantUnavailable as exc: return jsonify(error=str(exc)), 403 except AssistantError as exc: return jsonify(error=str(exc)), 502 return jsonify(reply=reply, agentic=is_agentic) @app.route("/assistant/confirm", methods=["POST"]) @login_required def assistant_confirm(): """Execute or cancel a pending Tier-1 tool action (Phase B+C). Request body: { token: str, confirmed: bool } Returns: { reply: str, agentic: bool } OR { pending_action: {...}, partial_reply: str, agentic: bool } when the resumed loop hits another Tier-1 tool. Security: the token is bound to the requesting user's email in _pending_consume(); any user who reached the agentic loop can confirm their own pending actions. """ from assistant import ( is_assistant_connected, resume_agentic, AssistantUnavailable, AssistantError, PendingActionRequired, ) user = session["user"] email = user["email"] is_admin = bool(user.get("togen_admin")) user_cfg = _load_user_settings(email) if not is_assistant_connected(email, is_admin, user_cfg): return jsonify(error="Assistant is not available."), 403 body = request.get_json(silent=True) or {} token = str(body.get("token") or "").strip() confirmed = bool(body.get("confirmed", False)) if not token: return jsonify(error="token is required."), 400 try: reply = resume_agentic(token, confirmed, user=user, user_settings=user_cfg) except PendingActionRequired as exc: return jsonify( pending_action={ "token": exc.token, "tool": exc.tool, "display_name": exc.display_name, "display_args": exc.display_args, }, partial_reply=exc.partial_reply, agentic=True, ) except AssistantUnavailable as exc: return jsonify(error=str(exc)), 403 except AssistantError as exc: return jsonify(error=str(exc)), 502 return jsonify(reply=reply, agentic=True) # --------------------------------------------------------------------------- # Auditor — Assistant tool-use audit review (DVI-558) # --------------------------------------------------------------------------- @app.route("/auditor/entries") @togen_admin_required def auditor_entries(): """Return Assistant tool-use audit log entries. Admin only. Query params: limit: max rows to return (default 100, max 500) offset: skip N most-recent rows (default 0) failures_only: '1' to return only non-success outcomes tool: exact tool name filter user: substring match on user email (case-insensitive) """ from assistant import TOOL_AUDIT_LOG try: limit = min(int(request.args.get("limit", 100)), 500) offset = max(int(request.args.get("offset", 0)), 0) except (TypeError, ValueError): return jsonify(error="Invalid limit or offset."), 400 failures_only = request.args.get("failures_only", "") == "1" f_tool = request.args.get("tool", "").strip() f_user = request.args.get("user", "").strip().lower() entries: list[dict] = [] if TOOL_AUDIT_LOG.is_file(): try: with TOOL_AUDIT_LOG.open(encoding="utf-8") as fh: for line in fh: line = line.strip() if not line: continue try: entry = json.loads(line) except json.JSONDecodeError: continue if failures_only and str(entry.get("outcome", "")) == "success": continue if f_tool and entry.get("tool") != f_tool: continue if f_user and f_user not in str(entry.get("user", "")).lower(): continue entries.append(entry) except OSError as exc: return jsonify(error=f"Could not read audit log: {exc}"), 500 # Reverse-chronological; paginate entries.reverse() total = len(entries) page = entries[offset:offset + limit] return jsonify(entries=page, total=total, offset=offset, limit=limit) # Friendly tool label for a print job's ``source`` (the originating tool's Print # button). New sources should be added here so the Auditor names them clearly. _PRINT_SOURCE_LABELS = { "shared": "Carrier Sheet / File Browser", "wor": "Work Order Admin", "prodrun": "Production Run", "mtslabels": "MTS Labels", } @app.route("/auditor/print-history") @togen_admin_required def auditor_print_history(): """Return Print-system job history for the Auditor → Printing tab. Admin only. Every Print button funnels through ``POST /print/jobs``, which records the job in ``print_jobs``; this endpoint is the read side of that audit trail. Query params: limit: max rows (default 100, max 500) offset: skip N most-recent rows (default 0) user: substring match on creator email (case-insensitive) source: exact source filter (shared|wor|prodrun|mtslabels) """ try: limit = min(int(request.args.get("limit", 100)), 500) offset = max(int(request.args.get("offset", 0)), 0) except (TypeError, ValueError): return jsonify(error="Invalid limit or offset."), 400 f_user = request.args.get("user", "").strip().lower() f_source = request.args.get("source", "").strip() where, params = [], [] if f_user: where.append("LOWER(j.created_by) LIKE ?") params.append(f"%{f_user}%") if f_source: where.append("j.source = ?") params.append(f_source) clause = (" WHERE " + " AND ".join(where)) if where else "" conn = _print_db() try: total = conn.execute( "SELECT COUNT(*) AS n FROM print_jobs j" + clause, params ).fetchone()["n"] rows = conn.execute( "SELECT j.id, j.created_by, j.source, j.file_ref, j.published_name," " j.kind, j.copies, j.created_at, a.hostname AS machine_name," " c.delivered_at, c.acked_at, c.acked_status, c.error_message" " FROM print_jobs j" " JOIN print_commands c ON c.id = j.id" " LEFT JOIN print_agents a ON a.machine_id = j.machine_id" + clause + " ORDER BY j.created_at DESC LIMIT ? OFFSET ?", params + [limit, offset], ).fetchall() finally: conn.close() jobs = [] for r in rows: src = r["source"] or "" jobs.append({ "id": r["id"], "user": r["created_by"] or "", "source": src, "source_label": _PRINT_SOURCE_LABELS.get(src, src or "—"), "file_ref": r["file_ref"] or "", "printer": r["published_name"] or "", "machine_name": r["machine_name"] or "", "kind": r["kind"], "copies": r["copies"], "created_at": r["created_at"], "status": _print_job_status(r), "error_message": r["error_message"] or None, }) return jsonify(jobs=jobs, total=total, offset=offset, limit=limit) # --------------------------------------------------------------------------- # Simple QR Creator (DVI-270) # --------------------------------------------------------------------------- @app.route("/simple-qr/generate", methods=["POST"]) @login_required def simple_qr_generate(): """Generate a QR code PNG from a URL string.""" body = request.get_json(silent=True) or {} url = (body.get("url") or "").strip() if not url: return jsonify(error="URL is required."), 400 try: qr_obj = _qrcode_mod.QRCode( error_correction=_qrcode_mod.constants.ERROR_CORRECT_M, box_size=10, border=2, ) qr_obj.add_data(url) qr_obj.make(fit=True) img = qr_obj.make_image(fill_color="black", back_color="white").convert("RGB") img = img.resize((400, 400), _pil_image.LANCZOS) buf = io.BytesIO() img.save(buf, format="PNG") buf.seek(0) img_b64 = base64.b64encode(buf.read()).decode("ascii") return jsonify(ok=True, image=img_b64) except Exception as exc: return jsonify(error=f"QR generation failed: {exc}"), 500 @app.route("/fb/email-link", methods=["POST"]) @login_required def fb_email_link(): """Email a document link via Microsoft Graph API.""" body = request.get_json(silent=True) or {} to_email = (body.get("email") or "").strip() url = (body.get("url") or "").strip() name = (body.get("name") or "Document").strip() if not to_email: return jsonify(error="Email address is required."), 400 if not url: return jsonify(error="No document URL provided."), 400 if not AZURE_MAIL_FROM: return jsonify(error="Email sender is not configured. Set AZURE_MAIL_FROM."), 500 app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire mail token: {err}"), 500 subject = f"Document — {name}" graph_body = { "message": { "subject": subject, "body": { "contentType": "Text", "content": ( f"A document has been shared with you:\n\n" f"{name}\n{url}\n\n" "This email was sent from the ICAST Togen application." ), }, "toRecipients": [ {"emailAddress": {"address": to_email}}, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code not in (200, 202): detail = resp.json().get("error", {}).get("message", resp.text[:200]) return jsonify(error=f"Graph API error: {detail}"), 502 return jsonify(ok=True) # --------------------------------------------------------------------------- # Email tool (DVI-470) — read the signed-in user's SSO mailbox via Microsoft # Graph and surface attachments for save/share/workflow actions. # # Reads use the SAME app-only (client-credentials) token already used for # sendMail. Reading another user's mailbox requires the Azure app registration # to hold the Graph *application* permission ``Mail.Read`` (admin consent). If # that grant is missing Graph returns 403 and these routes degrade to an # explicit "mailbox access not granted" state rather than failing hard. # --------------------------------------------------------------------------- _EMAIL_INBOX_TOP = 25 _GRAPH_BASE = "https://graph.microsoft.com/v1.0" # Attachment extensions a Togen workflow can consume (mirror of the frontend # _EMAIL_DOC_EXTS + xlsx/csv, i.e. anything that gets a "Workflow" button). _EMAIL_WORKFLOW_EXTS = {"pdf", "doc", "docx", "png", "jpg", "jpeg", "xlsx", "csv"} def _email_graph_token(): """Acquire the shared app-only Graph token, or None if unavailable.""" if not (_AZ_CLIENT_ID and _AZ_SECRET): return None try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) return result.get("access_token") except Exception: app.logger.exception("Email: Graph token acquisition error") return None def _email_current_mailbox(): """Return the signed-in user's mailbox address from the session.""" return (session.get("user") or {}).get("email", "") or "" def _email_fetch_attachment(token, mailbox, message_id, attachment_id): """Fetch one attachment. Returns (name, content_type, raw_bytes) or (None, None, None).""" try: resp = _http_requests.get( f"{_GRAPH_BASE}/users/{mailbox}/messages/{message_id}/attachments/{attachment_id}", headers={"Authorization": f"Bearer {token}"}, timeout=60, ) if resp.status_code != 200: app.logger.error("Email attachment fetch failed (%s): %s", resp.status_code, resp.text[:300]) return None, None, None data = resp.json() content_b64 = data.get("contentBytes") if not content_b64: return None, None, None return (data.get("name") or "attachment", data.get("contentType") or "application/octet-stream", base64.b64decode(content_b64)) except Exception: app.logger.exception("Email attachment fetch error") return None, None, None @app.route("/email/messages") @login_required def email_messages(): """Return the signed-in user's recent inbox messages (attachments expanded).""" mailbox = _email_current_mailbox() if not mailbox: return jsonify(error="No mailbox associated with this session."), 400 token = _email_graph_token() if not token: return jsonify(error="Mail service is not configured.", available=False, reason="config"), 200 params = { "$top": _EMAIL_INBOX_TOP, "$orderby": "receivedDateTime desc", "$select": "id,subject,from,receivedDateTime,isRead,hasAttachments,bodyPreview", "$expand": "attachments($select=id,name,contentType,size,isInline)", } try: resp = _http_requests.get( f"{_GRAPH_BASE}/users/{mailbox}/mailFolders/inbox/messages", headers={"Authorization": f"Bearer {token}"}, params=params, timeout=60, ) except Exception: app.logger.exception("Email messages fetch error") return jsonify(error="Could not reach the mail service."), 502 if resp.status_code == 403: return jsonify(available=False, reason="permission", mailbox=mailbox, error="Mailbox access has not been granted. An administrator " "must grant the Mail.Read application permission to the " "Togen app registration in Azure AD."), 200 if resp.status_code != 200: detail = resp.json().get("error", {}).get("message", resp.text[:200]) if resp.text else resp.status_code return jsonify(error=f"Graph API error: {detail}"), 502 messages = [] unread = 0 for m in resp.json().get("value", []): if not m.get("isRead", True): unread += 1 atts = [ { "id": a.get("id"), "name": a.get("name") or "attachment", "size": a.get("size", 0), "contentType": a.get("contentType") or "", } for a in (m.get("attachments") or []) if not a.get("isInline") ] messages.append({ "id": m.get("id"), "subject": m.get("subject") or "(no subject)", "from": ((m.get("from") or {}).get("emailAddress") or {}).get("address", ""), "from_name": ((m.get("from") or {}).get("emailAddress") or {}).get("name", ""), "received": m.get("receivedDateTime"), "is_read": m.get("isRead", True), "preview": (m.get("bodyPreview") or "")[:200], "attachments": atts, "has_attachments": bool(atts), }) return jsonify(available=True, mailbox=mailbox, messages=messages, unread=unread) @app.route("/email/unread-count") @login_required def email_unread_count(): """Counts for the Tools page tile badges: total unread, unread with attachments, and unread with a workflow-supported attachment (DVI-635).""" mailbox = _email_current_mailbox() if not mailbox: return jsonify(count=0, available=False) token = _email_graph_token() if not token: return jsonify(count=0, available=False) headers = {"Authorization": f"Bearer {token}"} try: resp = _http_requests.get( f"{_GRAPH_BASE}/users/{mailbox}/mailFolders/inbox", headers=headers, params={"$select": "unreadItemCount"}, timeout=30, ) except Exception: return jsonify(count=0, available=False) if resp.status_code != 200: return jsonify(count=0, available=False, reason="permission" if resp.status_code == 403 else "error") total = resp.json().get("unreadItemCount", 0) # Inspect unread messages that carry attachments to break the badge into # "with attachments" and "with a workflow-supported attachment". Inline # attachments are ignored to match the inbox/attachments views. unread_attach = 0 unread_workflow = 0 try: a_resp = _http_requests.get( f"{_GRAPH_BASE}/users/{mailbox}/mailFolders/inbox/messages", headers=headers, params={ "$filter": "isRead eq false and hasAttachments eq true", "$top": 100, "$select": "id", "$expand": "attachments($select=name,isInline)", }, timeout=45, ) if a_resp.status_code == 200: for m in a_resp.json().get("value", []): names = [a.get("name") or "" for a in (m.get("attachments") or []) if not a.get("isInline")] if not names: continue unread_attach += 1 if any(name.rsplit(".", 1)[-1].lower() in _EMAIL_WORKFLOW_EXTS for name in names if "." in name): unread_workflow += 1 except Exception: app.logger.exception("Email unread-count attachment scan error") return jsonify(count=total, attachments=unread_attach, workflow=unread_workflow, available=True) @app.route("/email/attachment") @login_required def email_attachment(): """Stream an attachment's bytes (inline preview / direct download).""" mailbox = _email_current_mailbox() message_id = request.args.get("message_id", "") attachment_id = request.args.get("attachment_id", "") if not (mailbox and message_id and attachment_id): return jsonify(error="message_id and attachment_id are required."), 400 token = _email_graph_token() if not token: return jsonify(error="Mail service is not configured."), 502 name, content_type, raw = _email_fetch_attachment(token, mailbox, message_id, attachment_id) if raw is None: return jsonify(error="Attachment could not be retrieved."), 404 as_attach = request.args.get("download") == "1" return send_file(io.BytesIO(raw), mimetype=content_type, as_attachment=as_attach, download_name=name) @app.route("/email/save-attachment", methods=["POST"]) @login_required def email_save_attachment(): """Save an email attachment into the shared Projects tree (File Browser root).""" body = request.get_json(silent=True) or {} message_id = (body.get("message_id") or "").strip() attachment_id = (body.get("attachment_id") or "").strip() dest_rel = (body.get("dest_path") or "").strip() filename = (body.get("filename") or "").strip() if not (message_id and attachment_id): return jsonify(error="message_id and attachment_id are required."), 400 dest_dir = _fb_safe_path(dest_rel) if dest_rel else _FB_ROOT if dest_dir is None or not dest_dir.is_dir(): return jsonify(error="Invalid destination folder."), 400 mailbox = _email_current_mailbox() token = _email_graph_token() if not token: return jsonify(error="Mail service is not configured."), 502 name, _ct, raw = _email_fetch_attachment(token, mailbox, message_id, attachment_id) if raw is None: return jsonify(error="Attachment could not be retrieved."), 404 safe_name = secure_filename(filename or name) or "attachment" dest = dest_dir / safe_name if dest.exists(): stem, ext = os.path.splitext(safe_name) dest = dest_dir / f"{stem}_{_datetime_type.now().strftime('%H%M%S')}{ext}" try: dest.write_bytes(raw) except OSError: app.logger.exception("Email save-attachment write error") return jsonify(error="Could not write the file to the destination."), 500 saved_rel = str(dest.relative_to(_FB_ROOT)) return jsonify(ok=True, saved_path=saved_rel, filename=dest.name) @app.route("/email/share-attachment", methods=["POST"]) @login_required def email_share_attachment(): """Forward an email attachment to a recipient via Graph sendMail.""" body = request.get_json(silent=True) or {} message_id = (body.get("message_id") or "").strip() attachment_id = (body.get("attachment_id") or "").strip() to_email = (body.get("email") or "").strip() if not (message_id and attachment_id): return jsonify(error="message_id and attachment_id are required."), 400 if not to_email: return jsonify(error="A recipient email address is required."), 400 if not AZURE_MAIL_FROM: return jsonify(error="Email sender is not configured. Set AZURE_MAIL_FROM."), 500 mailbox = _email_current_mailbox() token = _email_graph_token() if not token: return jsonify(error="Mail service is not configured."), 502 name, content_type, raw = _email_fetch_attachment(token, mailbox, message_id, attachment_id) if raw is None: return jsonify(error="Attachment could not be retrieved."), 404 sender = _email_current_mailbox() or "a colleague" html = notifications.render_email_shell( "A file was shared with you", f"

    {html.escape(sender)} shared the file {html.escape(name)} " "with you from the ICAST Togen Email tool.

    ", subtitle="Shared attachment", ) ok = notifications.send_email_with_attachment( to_email, f"Shared file — {name}", html, filename=name, content=raw, content_type=content_type) if not ok: return jsonify(error="The mail service rejected the message."), 502 return jsonify(ok=True) @app.route("/email/add-to-manuals", methods=["POST"]) @login_required def email_add_to_manuals(): """Save an email attachment as an Equipment Manual associated with a name.""" body = request.get_json(silent=True) or {} message_id = (body.get("message_id") or "").strip() attachment_id = (body.get("attachment_id") or "").strip() equipment = (body.get("equipment") or "").strip() if not (message_id and attachment_id): return jsonify(error="message_id and attachment_id are required."), 400 if not equipment: return jsonify(error="Equipment / Asset / System name is required."), 400 mailbox = _email_current_mailbox() token = _email_graph_token() if not token: return jsonify(error="Mail service is not configured."), 502 name, _ct, raw = _email_fetch_attachment(token, mailbox, message_id, attachment_id) if raw is None: return jsonify(error="Attachment could not be retrieved."), 404 ext = os.path.splitext(name)[1].lower() if ext not in ALLOWED_MANUAL_EXTS: allowed = ", ".join(sorted(ALLOWED_MANUAL_EXTS)) return jsonify(error=f"Unsupported manual file type. Allowed: {allowed}"), 400 WOR_MANUALS_DIR.mkdir(parents=True, exist_ok=True) base = secure_filename(name) or ("manual" + ext) stored = f"{_datetime_type.now().strftime('%Y%m%d_%H%M%S')}_{base}" try: (WOR_MANUALS_DIR / stored).write_bytes(raw) except OSError: app.logger.exception("Email add-to-manuals write error") return jsonify(error="Could not save the manual."), 500 _manuals_index_set(equipment, stored, name, session.get("user") or {}) _enqueue_manual_ocr(stored, name) return jsonify(ok=True, filename=stored, original_name=name, equipment=equipment, url=_manual_viewer_url(stored)) # --------------------------------------------------------------------------- # Calendar tool (DVI-556) — surface the signed-in user's Microsoft 365 calendar # events in the Calendar tool. Uses the SAME app-only Graph token as the Email # tool; reading another user's calendar requires the Azure app registration to # hold the Graph application permission ``Calendars.Read`` (admin consent). If # that grant is missing Graph returns 403 and the route degrades to an explicit # "calendar access not granted" state rather than failing hard. # --------------------------------------------------------------------------- @app.route("/calendar/events") @login_required def calendar_events(): """Return the signed-in user's M365 calendar events within a window. Query params ``start`` and ``end`` are ISO-8601 datetimes bounding the requested range (the client passes a padded window around the visible view). """ mailbox = _email_current_mailbox() if not mailbox: return jsonify(error="No mailbox associated with this session.", available=False, reason="config"), 200 start = (request.args.get("start") or "").strip() end = (request.args.get("end") or "").strip() if not (start and end): return jsonify(error="start and end are required."), 400 token = _email_graph_token() if not token: return jsonify(available=False, reason="config", error="Calendar service is not configured."), 200 params = { "startDateTime": start, "endDateTime": end, "$select": "id,subject,start,end,isAllDay,location,organizer,bodyPreview", "$orderby": "start/dateTime", "$top": 200, } try: resp = _http_requests.get( f"{_GRAPH_BASE}/users/{mailbox}/calendarView", headers={ "Authorization": f"Bearer {token}", # Return start/end in UTC for consistent client-side parsing. "Prefer": 'outlook.timezone="UTC"', }, params=params, timeout=60, ) except Exception: app.logger.exception("Calendar events fetch error") return jsonify(error="Could not reach the calendar service."), 502 if resp.status_code == 403: return jsonify(available=False, reason="permission", mailbox=mailbox, error="Calendar access has not been granted. An administrator " "must grant the Calendars.Read application permission to " "the Togen app registration in Azure AD."), 200 if resp.status_code != 200: detail = (resp.json().get("error", {}).get("message", resp.text[:200]) if resp.text else resp.status_code) return jsonify(error=f"Graph API error: {detail}"), 502 events = [] for ev in resp.json().get("value", []): events.append({ "id": ev.get("id"), "subject": ev.get("subject") or "(no title)", "start": (ev.get("start") or {}).get("dateTime"), "end": (ev.get("end") or {}).get("dateTime"), "all_day": bool(ev.get("isAllDay")), "location": ((ev.get("location") or {}).get("displayName") or ""), "organizer": (((ev.get("organizer") or {}).get("emailAddress") or {}).get("name") or ""), "preview": (ev.get("bodyPreview") or "")[:200], "source": "microsoft", }) return jsonify(available=True, mailbox=mailbox, events=events) @app.route("/calendar/wo-events") @login_required def calendar_wo_events(): """Work Order date events for the Calendar tool (DVI-630). Returns two kinds of all-day events within the requested window: * recurring schedules — one marker per due date across the window, and * active (new / in-progress) WOs that carry a Desired Completion Date or Date Promised. Visible only to Work Order Admins and Togen Admins (the ``togen_admin`` role already maps to both the iCast and Voyage Togen-Admin Azure groups), matching the PM-access gate. Non-privileged users get ``available:false`` so the client never renders the Work Orders filter for them. """ user = session.get("user") or {} if not _user_can_access_pm(user): return jsonify(available=False, reason="permission", events=[]), 200 start = (request.args.get("start") or "").strip() end = (request.args.get("end") or "").strip() if not (start and end): return jsonify(error="start and end are required."), 400 try: win_start = _datetime_type.fromisoformat(start.replace("Z", "+00:00")).date() win_end = _datetime_type.fromisoformat(end.replace("Z", "+00:00")).date() except (ValueError, TypeError): return jsonify(error="Invalid start/end."), 400 _type_label = { "maintenance": "Maintenance", "mobilemachine": "Mobile Machine", "safety": "Safety", "preventive": "Preventive", } events = [] # 1) Recurring schedules — a marker on each due date in the window. for source_filename, rec in _load_recurring_registry().items(): if not rec.get("enabled", True): continue wor_type = rec.get("wor_type", "maintenance") template = rec.get("template_data") or {} title = _wor_display_title(source_filename, wor_type, template) for run_date in _recurring_runs_in_window( rec.get("start_date", ""), rec.get("frequency_days", 0), rec.get("end_date", ""), win_start, win_end): iso = run_date.strftime("%Y-%m-%d") events.append({ "id": f"recurring:{source_filename}:{iso}", "subject": title, "start": iso, "end": iso, "all_day": True, "location": template.get("location") or template.get("building") or "", "wo_file": source_filename, "wo_kind": "recurring", "wo_date_type": "Recurring due", "wor_type": wor_type, "wor_type_label": _type_label.get(wor_type, wor_type), "source": "workorder", }) # 2) Active WOs with a Desired Completion Date or Date Promised. for wor_type in ("maintenance", "mobilemachine", "safety", "preventive"): dirs = _wor_dest_dirs(wor_type) for status_key in ("new", "in_progress"): directory = dirs.get(status_key) if not directory or not directory.is_dir(): continue for f in directory.glob("*.pdf"): sidecar = f.with_suffix(".json") if not sidecar.is_file(): continue try: sd = json.loads(sidecar.read_text()) or {} except (ValueError, OSError): continue title = _wor_display_title(f.name, wor_type, sd) for field, label in (("desired_completion_date", "Desired completion"), ("date_promised", "Date promised")): val = (sd.get(field) or "").strip() if not val: continue try: d = _datetime_type.strptime(val[:10], "%Y-%m-%d").date() except (ValueError, TypeError): continue if d < win_start or d > win_end: continue events.append({ "id": f"{field}:{f.name}", "subject": title, "start": val[:10], "end": val[:10], "all_day": True, "location": sd.get("location") or sd.get("building") or "", "wo_file": f.name, "wo_kind": "due", "wo_date_type": label, "wor_type": wor_type, "wor_type_label": _type_label.get(wor_type, wor_type), "source": "workorder", }) return jsonify(available=True, events=events) @app.route("/simple-qr/email", methods=["POST"]) @login_required def simple_qr_email(): """Email a QR code PNG as an attachment via Microsoft Graph API.""" body = request.get_json(silent=True) or {} to_email = (body.get("email") or "").strip() image_b64 = (body.get("image") or "").strip() url = (body.get("url") or "").strip() if not to_email: return jsonify(error="Email address is required."), 400 if not image_b64: return jsonify(error="No QR code image provided."), 400 if not AZURE_MAIL_FROM: return jsonify(error="Email sender is not configured. Set AZURE_MAIL_FROM."), 500 app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire mail token: {err}"), 500 subject = f"QR Code — {url}" if url else "QR Code" graph_body = { "message": { "subject": subject, "body": { "contentType": "Text", "content": ( f"Please find the attached QR code for:\n{url}\n\n" "This email was sent from the ICAST Togen application." ), }, "toRecipients": [ {"emailAddress": {"address": to_email}}, ], "attachments": [ { "@odata.type": "#microsoft.graph.fileAttachment", "name": "qr-code.png", "contentType": "image/png", "contentBytes": image_b64, }, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code not in (200, 202): detail = resp.json().get("error", {}).get("message", resp.text[:200]) return jsonify(error=f"Graph API error: {detail}"), 502 return jsonify(ok=True) # --------------------------------------------------------------------------- # MTS Label Creator routes # --------------------------------------------------------------------------- @app.route("/mts/products") @mts_label_required def mts_products_list(): """Return a list of all stock products sorted by product code.""" products = [] if STOCK_PRODUCTS_DIR.is_dir(): for f in STOCK_PRODUCTS_DIR.glob("*.json"): try: data = json.loads(f.read_text(encoding="utf-8")) has_drawing = (STOCK_DRAWINGS_DIR / f"{f.stem}.pdf").is_file() products.append({ "product_code": data.get("product_code", f.stem), "description": data.get("description", ""), "weight": data.get("weight", ""), "has_drawing": has_drawing, }) except (json.JSONDecodeError, OSError): continue products.sort(key=lambda p: p["product_code"].lower()) return jsonify(products=products) @app.route("/mts/product/") @mts_label_required def mts_product_get(product_code): """Return stored product data (description, weight) for a product code.""" safe = secure_filename(product_code) if not safe: return jsonify(error="Invalid product code"), 400 json_path = STOCK_PRODUCTS_DIR / f"{safe}.json" if not json_path.is_file(): return jsonify(found=False) try: data = json.loads(json_path.read_text(encoding="utf-8")) except Exception: return jsonify(found=False) has_drawing = (STOCK_DRAWINGS_DIR / f"{safe}.pdf").is_file() data["found"] = True data["has_drawing"] = has_drawing return jsonify(data) @app.route("/mts/product", methods=["POST"]) @mts_label_required def mts_product_save(): """Save / update product data for a product code.""" d = request.get_json(force=True) code = d.get("product_code", "").strip() safe = secure_filename(code) if not safe: return jsonify(error="Product code is required"), 400 STOCK_PRODUCTS_DIR.mkdir(parents=True, exist_ok=True) json_path = STOCK_PRODUCTS_DIR / f"{safe}.json" payload = { "product_code": code, "description": d.get("description", "").strip(), "weight": d.get("weight", "").strip(), } json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") return jsonify(ok=True) @app.route("/mts/drawing/", methods=["POST"]) @mts_label_required def mts_drawing_upload(product_code): """Upload a drawing PDF for a product code.""" safe = secure_filename(product_code) if not safe: return jsonify(error="Invalid product code"), 400 if "drawing" not in request.files: return jsonify(error="No drawing file provided"), 400 f = request.files["drawing"] if not f.filename or not f.filename.lower().endswith(".pdf"): return jsonify(error="Drawing must be a PDF file"), 400 STOCK_DRAWINGS_DIR.mkdir(parents=True, exist_ok=True) dest = STOCK_DRAWINGS_DIR / f"{safe}.pdf" f.save(str(dest)) return jsonify(ok=True) @app.route("/mts/drawing/") @mts_label_required def mts_drawing_get(product_code): """Serve the drawing PDF for a product code.""" safe = secure_filename(product_code) if not safe: abort(400) path = STOCK_DRAWINGS_DIR / f"{safe}.pdf" if not path.is_file(): abort(404) return send_file(str(path), mimetype="application/pdf") @app.route("/mts/label", methods=["POST"]) @mts_label_required def mts_label_create(): """Generate an MTS label PDF and return it for preview.""" d = request.get_json(force=True) code = d.get("product_code", "").strip() description = d.get("description", "").strip() weight = d.get("weight", "").strip() label_date = d.get("date", "").strip() created_by = session.get("user", {}).get("name", "Unknown") if not code or not description or not weight or not label_date: return jsonify(error="All fields are required"), 400 safe_code = secure_filename(code) if not safe_code: return jsonify(error="Invalid product code"), 400 safe_date = secure_filename(label_date) filename = f"{safe_date}_{safe_code}.pdf" MTS_LABELS_DIR.mkdir(parents=True, exist_ok=True) out_path = MTS_LABELS_DIR / filename drawing_path = STOCK_DRAWINGS_DIR / f"{safe_code}.pdf" has_drawing = drawing_path.is_file() qr_url = "" qr_tmp_path = None if has_drawing: prefix = _load_shared_url_prefix() qr_url = f"{prefix}/stockproducts/drawings/{quote(safe_code)}.pdf" try: qr_obj = _qrcode_mod.QRCode( error_correction=_qrcode_mod.constants.ERROR_CORRECT_M, box_size=10, border=2, ) qr_obj.add_data(qr_url) qr_obj.make(fit=True) qr_img = qr_obj.make_image(fill_color="black", back_color="white").convert("RGB") fd, qr_tmp_path = tempfile.mkstemp(suffix=".png") os.close(fd) qr_img.save(qr_tmp_path, format="PNG") except Exception as exc: print(f"[MTS] QR generation failed for {code}: {exc}", flush=True) qr_tmp_path = None try: # 7in x 12in label size required by label printer pdf = FPDF(orientation="P", unit="mm", format=(177.8, 304.8)) pdf.set_auto_page_break(auto=False) pdf.add_page() W = pdf.w # 177.8mm (7in) H = pdf.h # 304.8mm (12in) margin = 10 pw = W - 2 * margin # 157.8mm # --- Logo centered at top (large) --- logo_path = _TOGEN_APP_DIR / "logo-black.svg" logo_h = 72 logo_w = logo_h * (334 / 307) # ~78.3mm wide logo_x = (W - logo_w) / 2 if logo_path.is_file(): pdf.image(str(logo_path), x=logo_x, y=margin, h=logo_h) # --- Product code (auto-shrink to fit) above separator --- y = margin + logo_h + 5 font_size = 46 pdf.set_font("Helvetica", "B", font_size) while pdf.get_string_width(code) > pw - 4 and font_size > 14: font_size -= 2 pdf.set_font("Helvetica", "B", font_size) cell_h = max(font_size * 0.45, 14) pdf.set_xy(margin, y) pdf.cell(pw, cell_h, code, align="C") y += cell_h + 4 # --- Separator --- pdf.set_line_width(0.5) pdf.line(margin, y, W - margin, y) y += 8 # --- Info box geometry (date + created by, bottom-right) --- box_w = 80 box_h = 26 box_x = W - margin - box_w box_y = H - margin - box_h # --- Field rows (description and weight only) --- label_col_w = 36 val_col_w = pw - label_col_w row_h = 10 for heading, value in [("Description:", description), ("Weight:", weight)]: pdf.set_font("Helvetica", "B", 13) pdf.set_xy(margin, y) pdf.cell(label_col_w, row_h, heading, align="L") pdf.set_font("Helvetica", "B" if heading == "Weight:" else "", 13) pdf.set_xy(margin + label_col_w, y) pdf.multi_cell(val_col_w, row_h, value, align="L") y = pdf.get_y() + 2 # --- QR code centered in remaining space (if available) --- if qr_tmp_path: qr_bottom = box_y - 4 remaining = qr_bottom - y qr_size = min(80.0, remaining - 4) if qr_size > 20: qr_x = (W - qr_size) / 2 qr_y = y + (remaining - qr_size) / 2 pdf.image(qr_tmp_path, x=qr_x, y=qr_y, w=qr_size, h=qr_size) # --- Date / Created by (bottom-right, no box) --- pdf.set_font("Helvetica", "", 9) pdf.set_xy(box_x, box_y + 3) pdf.cell(box_w, 5, f"Date: {label_date}", align="R") pdf.set_xy(box_x, box_y + 10) pdf.cell(box_w, 5, f"Created by: {created_by}", align="R") pdf.output(str(out_path)) finally: if qr_tmp_path and os.path.exists(qr_tmp_path): os.unlink(qr_tmp_path) STOCK_PRODUCTS_DIR.mkdir(parents=True, exist_ok=True) prod_path = STOCK_PRODUCTS_DIR / f"{safe_code}.json" prod_data = { "product_code": code, "description": description, "weight": weight, "qr_url": qr_url, } prod_path.write_text(json.dumps(prod_data, indent=2), encoding="utf-8") if _is_history_capture_enabled("mts-label"): record_history( "mts-label", "created", actor=created_by, summary=f"MTS label created — {code}: {description}", ref=code, detail={"product_code": code, "weight": weight, "date": label_date, "filename": filename}, ) return jsonify(ok=True, filename=filename) @app.route("/mts/label/preview/") @mts_label_required def mts_label_preview(filename): """Serve a generated MTS label PDF for preview/download.""" safe = secure_filename(filename) if not safe: abort(400) path = MTS_LABELS_DIR / safe if not path.is_file(): abort(404) return send_file(str(path), mimetype="application/pdf") @app.route("/mts/email", methods=["POST"]) @mts_label_required def mts_email_label(): """Email an MTS label PDF as an attachment via Microsoft Graph API.""" body = request.get_json(silent=True) or {} to_email = (body.get("email") or "").strip() filename = (body.get("filename") or "").strip() if not to_email: return jsonify(error="Email address is required."), 400 if not filename: return jsonify(error="No label filename provided."), 400 safe = secure_filename(filename) if not safe: return jsonify(error="Invalid filename."), 400 path = MTS_LABELS_DIR / safe if not path.is_file(): return jsonify(error="Label file not found."), 404 if not AZURE_MAIL_FROM: return jsonify(error="Email sender is not configured. Set AZURE_MAIL_FROM."), 500 pdf_bytes = path.read_bytes() pdf_b64 = base64.b64encode(pdf_bytes).decode("ascii") app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire mail token: {err}"), 500 subject = f"MTS Label — {safe}" graph_body = { "message": { "subject": subject, "body": { "contentType": "Text", "content": ( f"Please find the attached MTS label: {safe}\n\n" "This email was sent from the ICAST Togen application." ), }, "toRecipients": [ {"emailAddress": {"address": to_email}}, ], "attachments": [ { "@odata.type": "#microsoft.graph.fileAttachment", "name": safe, "contentType": "application/pdf", "contentBytes": pdf_b64, }, ], }, "saveToSentItems": "false", } resp = _http_requests.post( f"https://graph.microsoft.com/v1.0/users/{AZURE_MAIL_FROM}/sendMail", headers={ "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", }, json=graph_body, timeout=30, ) if resp.status_code not in (200, 202): detail = resp.json().get("error", {}).get("message", resp.text[:200]) return jsonify(error=f"Graph API error: {detail}"), 502 return jsonify(ok=True) # --------------------------------------------------------------------------- # SCADA View — read temperature sensor logs from shared mailbox (DVI-300) # --------------------------------------------------------------------------- SCADA_MAILBOX = "scadalogs@icastinc.com" @app.route("/scada/logs") @scada_required def scada_logs(): """Fetch recent messages from the SCADA logs mailbox via Microsoft Graph API. Requires the Azure AD app registration to have the application-level Mail.Read permission (or Mail.ReadBasic) and admin consent for the target mailbox. The app already authenticates with client credentials for sendMail, so the same credential works here — the only addition is the Mail.Read (or Mail.ReadBasic.All) scope in Azure AD > App registrations > API permissions. """ app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire Graph API token: {err}"), 500 headers = { "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", } resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/users/{SCADA_MAILBOX}/messages" "?$top=50&$orderby=receivedDateTime desc" "&$select=subject,receivedDateTime,bodyPreview,body", headers=headers, timeout=30, ) if resp.status_code == 403: return jsonify( error="Mail.Read permission not yet granted for the SCADA mailbox. " "An Azure AD administrator must grant the Mail.Read (application) " "permission and admin-consent it for this app registration." ), 403 if resp.status_code != 200: detail = "" try: detail = resp.json().get("error", {}).get("message", resp.text[:300]) except Exception: detail = resp.text[:300] return jsonify(error=f"Graph API error ({resp.status_code}): {detail}"), 502 messages = resp.json().get("value", []) logs = [] for msg in messages: received = msg.get("receivedDateTime", "") if received: try: dt = _datetime_type.fromisoformat(received.replace("Z", "+00:00")) received = dt.strftime("%Y-%m-%d %H:%M") except Exception: pass body_content = "" body_obj = msg.get("body", {}) if body_obj.get("contentType") == "text": body_content = body_obj.get("content", "") else: body_content = msg.get("bodyPreview", "") logs.append({ "date": received, "subject": msg.get("subject", "(no subject)"), "body": body_content, }) return jsonify(logs=logs) def _parse_scada_body(body_text): """Parse a SCADA email body into (timestamp, [{label, value}, ...]).""" readings = [] timestamp = None for line in body_text.splitlines(): line = line.strip() if not line: continue if line.lower().startswith("time:"): timestamp = line.split(":", 1)[1].strip() continue if ":" in line: label, _, value = line.partition(":") label = label.strip() value = value.strip() if label and value: readings.append({"label": label, "value": value}) return timestamp, readings def _scada_message_body(msg): """Extract plain-text body content from a Graph message object.""" body_obj = msg.get("body", {}) if body_obj.get("contentType") == "text": return body_obj.get("content", "") return msg.get("bodyPreview", "") @app.route("/scada/status") @scada_required def scada_status(): """Parse the most recent SCADA email into structured sensor readings.""" app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire Graph API token: {err}"), 500 headers = { "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", } # Fetch a small recent window rather than just the newest message: the # mailbox now interleaves "X-400 - Log" daily-table emails with the # "X-400 - Trigger: ..." status emails (DVI-538), and status must come from # the latest Trigger email, not whatever happens to be on top. resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/users/{SCADA_MAILBOX}/messages" "?$top=25&$orderby=receivedDateTime desc" "&$select=subject,receivedDateTime,bodyPreview,body", headers=headers, timeout=30, ) if resp.status_code == 403: return jsonify( error="Mail.Read permission not yet granted for the SCADA mailbox." ), 403 if resp.status_code != 200: detail = "" try: detail = resp.json().get("error", {}).get("message", resp.text[:300]) except Exception: detail = resp.text[:300] return jsonify(error=f"Graph API error ({resp.status_code}): {detail}"), 502 messages = resp.json().get("value", []) trigger = next( (m for m in messages if scada_subject_kind(m.get("subject")) == "trigger"), None) if trigger is None: return jsonify(readings=[], time=None) timestamp, readings = _parse_scada_body(_scada_message_body(trigger)) return jsonify(readings=readings, time=timestamp) @app.route("/scada/history") @scada_required def scada_history(): """Build per-label time series from recent SCADA emails (Option A: derive on demand). Re-parses up to ~200 recent messages from the SCADA mailbox into {label: [{time, value}, ...]} sorted ascending by message time. No new storage or scheduler — depth is bounded by mailbox retention + $top. """ app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) if "access_token" not in token_result: err = token_result.get("error_description", token_result.get("error", "unknown")) return jsonify(error=f"Failed to acquire Graph API token: {err}"), 500 headers = { "Authorization": f"Bearer {token_result['access_token']}", "Content-Type": "application/json", } resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/users/{SCADA_MAILBOX}/messages" "?$top=200&$orderby=receivedDateTime desc" "&$select=subject,receivedDateTime,bodyPreview,body", headers=headers, timeout=30, ) if resp.status_code == 403: return jsonify( error="Mail.Read permission not yet granted for the SCADA mailbox." ), 403 if resp.status_code != 200: detail = "" try: detail = resp.json().get("error", {}).get("message", resp.text[:300]) except Exception: detail = resp.text[:300] return jsonify(error=f"Graph API error ({resp.status_code}): {detail}"), 502 messages = resp.json().get("value", []) # Both email types inform history (DVI-538): Trigger emails contribute one # point-in-time snapshot each, while a single "X-400 - Log" email contributes # a full day of 15-min interval rows. Sensor labels are normalized # ("K1 Temp(F)" -> "K1 Temp") so the two sources land on the same series. series = {} for msg in messages: kind = scada_subject_kind(msg.get("subject")) received = msg.get("receivedDateTime", "") time_label = received if received: try: dt = _datetime_type.fromisoformat(received.replace("Z", "+00:00")) time_label = dt.strftime("%Y-%m-%d %H:%M") except Exception: pass if kind == "trigger": _ts, readings = _parse_scada_body(_scada_message_body(msg)) point_time = _ts or time_label for r in readings: key = _scada_norm_sensor(r["label"]) series.setdefault(key, []).append( {"time": point_time, "value": r["value"]}) elif kind == "log": parsed = _parse_scada_log_email(_scada_log_body_text(msg)) cols = parsed["columns"] sensor_cols = cols[1:-1] if len(cols) >= 2 else [] for row in parsed["rows"]: dt = row.get("_dt") point_time = dt.strftime("%Y-%m-%d %H:%M") if dt else time_label for col in sensor_cols: val = (row.get(col, "") or "").strip() if not val or val.lower() == "x.x": continue series.setdefault(_scada_norm_sensor(col), []).append( {"time": point_time, "value": val}) # Order each series ascending by time (sources arrive newest-first and # interleave), keeping the chart monotonic. for points in series.values(): points.sort(key=lambda p: p["time"]) return jsonify(series=series, count=len(messages)) # --------------------------------------------------------------------------- # SCADA Reporting engine (DVI-482 / DVI-474 ST1) # --------------------------------------------------------------------------- # The engine was extracted into the self-contained ``scada_report`` module # (DVI-486) so the standalone report scheduler can reuse it without importing # app.py (which starts background loops at import time). The names below are # imported so the existing /scada/report[/download] call sites are unchanged. from scada_report import ( # noqa: E402 _build_scada_report, _parse_scada_log_email, _scada_log_body_text, _scada_norm_sensor, _scada_report_workbook, scada_subject_kind, ) def _scada_parse_report_args(req): """Parse date_from/date_to (YYYY-MM-DD) and sensors (CSV or repeated) from a request. Returns (date_from, date_to, sensors, error).""" def parse_date(raw): if not raw: return None return _datetime_type.strptime(raw, "%Y-%m-%d").date() try: date_from = parse_date(req.args.get("date_from", "").strip()) date_to = parse_date(req.args.get("date_to", "").strip()) except ValueError: return None, None, None, "Invalid date; expected YYYY-MM-DD." if date_from and date_to and date_from > date_to: return None, None, None, "date_from must be on or before date_to." sensors = req.args.getlist("sensors") if len(sensors) == 1 and "," in sensors[0]: sensors = sensors[0].split(",") sensors = [s.strip() for s in sensors if s.strip()] return date_from, date_to, sensors, None def _scada_report_error_response(err): """Map a _build_scada_report error string to a JSON Flask response.""" if err == "403": return jsonify( error="Mail.Read permission not yet granted for the SCADA mailbox." ), 403 return jsonify(error=f"SCADA mailbox fetch failed: {err}"), 502 @app.route("/scada/report") @scada_required def scada_report(): """Preview JSON for the SCADA report: per-day tables + available sensors + earliest available date, honoring the {date_from, date_to, sensors[]} filter.""" date_from, date_to, sensors, err = _scada_parse_report_args(request) if err: return jsonify(error=err), 400 report = _build_scada_report(date_from, date_to, sensors) if report["error"]: return _scada_report_error_response(report["error"]) return jsonify( days=report["days"], columns=report["columns"], available_sensors=report["available_sensors"], earliest_date=report["earliest_date"], row_count=report["row_count"], ) @app.route("/scada/report/download") @scada_required def scada_report_download(): """Stream the SCADA report as an XLSX (one tab per day).""" date_from, date_to, sensors, err = _scada_parse_report_args(request) if err: return jsonify(error=err), 400 report = _build_scada_report(date_from, date_to, sensors) if report["error"]: return _scada_report_error_response(report["error"]) buf = _scada_report_workbook(report) def fmt(d): return f"{d.month}.{d.day}.{d.strftime('%y')}" if d else "all" name = f"{fmt(date_from)} - {fmt(date_to)}.xlsx" return send_file( buf, as_attachment=True, download_name=name, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) @app.route("/scada/reports") @scada_required def scada_reports_list(): """List produced XLSX report files (written by the report scheduler), newest first. Used by the File Browser SCADA tab.""" reports = [] if SCADA_REPORTS_DIR.is_dir(): for p in SCADA_REPORTS_DIR.glob("*.xlsx"): if not p.is_file(): continue try: st = p.stat() except OSError: continue reports.append({ "name": p.name, "size": st.st_size, "modified": _datetime_type.fromtimestamp(st.st_mtime).strftime("%Y-%m-%d %H:%M"), "ts": st.st_mtime, }) reports.sort(key=lambda r: r["ts"], reverse=True) return jsonify(reports=reports) @app.route("/scada/reports/download") @scada_required def scada_reports_download(): """Stream a single persisted XLSX report by filename (path-traversal safe).""" name = request.args.get("name", "") # Reject any path component — only a bare filename in the reports dir is allowed. if not name or name != Path(name).name or not name.lower().endswith(".xlsx"): return jsonify(error="Invalid report name."), 400 path = (SCADA_REPORTS_DIR / name).resolve() try: path.relative_to(SCADA_REPORTS_DIR.resolve()) except ValueError: return jsonify(error="Invalid report name."), 400 if not path.is_file(): return jsonify(error="Report not found."), 404 return send_file( path, as_attachment=True, download_name=name, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) # Data Collector defaults (DVI-465): how often the collector polls the SCADA # mailbox for new readings, and how long without a new reading before a stale- # data notification should fire. Both in minutes. SCADA_COLLECTOR_DEFAULTS = {"check_interval_minutes": 15, "stale_after_minutes": 60} def _load_scada_config(): """Return shared SCADA config: {ranges: {label: {low, high}}, enabled: {label: bool}, aliases: {label: str}, reporting: {label: bool}, notify: {label: bool}, collector: {check_interval_minutes, stale_after_minutes}}.""" defaults = {"ranges": {}, "enabled": {}, "aliases": {}, "reporting": {}, "notify": {}, "collector": dict(SCADA_COLLECTOR_DEFAULTS)} if SCADA_CONFIG_FILE.is_file(): try: saved = json.loads(SCADA_CONFIG_FILE.read_text()) if isinstance(saved.get("ranges"), dict): defaults["ranges"] = saved["ranges"] if isinstance(saved.get("enabled"), dict): defaults["enabled"] = saved["enabled"] if isinstance(saved.get("aliases"), dict): defaults["aliases"] = saved["aliases"] if isinstance(saved.get("reporting"), dict): defaults["reporting"] = saved["reporting"] if isinstance(saved.get("notify"), dict): defaults["notify"] = saved["notify"] if isinstance(saved.get("collector"), dict): defaults["collector"].update({ k: saved["collector"][k] for k in SCADA_COLLECTOR_DEFAULTS if k in saved["collector"] }) except (json.JSONDecodeError, OSError): pass return defaults def _save_scada_config(config): """Persist shared SCADA config to disk.""" SCADA_CONFIG_FILE.write_text(json.dumps(config, indent=2)) @app.route("/scada/config") @scada_required def scada_config_get(): """Return the shared SCADA config (ranges + enabled flags).""" return jsonify(config=_load_scada_config()) @app.route("/scada/config", methods=["POST"]) @scada_required def scada_config_post(): """Save shared SCADA config (ranges + enabled flags).""" body = request.get_json(silent=True) or {} config = _load_scada_config() # Bulk override (DVI-483): apply one temp/humidity range + notify flag to all # discovered sensors, overwriting their individual settings. if body.get("bulk_override"): labels = body.get("labels") if not isinstance(labels, list): labels = [] temp = body.get("temp") if isinstance(body.get("temp"), dict) else {} humidity = body.get("humidity") if isinstance(body.get("humidity"), dict) else {} notify_enabled = bool(body.get("notify_enabled")) ranges = config.get("ranges") or {} notify = config.get("notify") or {} for label in labels: if not isinstance(label, str) or not _scada_is_sensor_label(label): continue rng = temp if "Temp" in label else humidity ranges[label] = {"low": rng.get("low", ""), "high": rng.get("high", "")} notify[label] = notify_enabled config["ranges"] = ranges config["notify"] = notify _save_scada_config(config) return jsonify(ok=True) if "ranges" in body and isinstance(body["ranges"], dict): config["ranges"] = body["ranges"] if "enabled" in body and isinstance(body["enabled"], dict): config["enabled"] = body["enabled"] if "aliases" in body and isinstance(body["aliases"], dict): config["aliases"] = body["aliases"] if "reporting" in body and isinstance(body["reporting"], dict): config["reporting"] = body["reporting"] if "notify" in body and isinstance(body["notify"], dict): config["notify"] = body["notify"] if "collector" in body and isinstance(body["collector"], dict): collector = config.get("collector") or dict(SCADA_COLLECTOR_DEFAULTS) for key in SCADA_COLLECTOR_DEFAULTS: if key in body["collector"]: try: val = int(body["collector"][key]) except (TypeError, ValueError): continue collector[key] = max(1, val) config["collector"] = collector _save_scada_config(config) return jsonify(ok=True) # --------------------------------------------------------------------------- # SCADA report schedules (DVI-484) — saved report definitions that ST4's # scheduler executes on a cadence and (optionally) emails. Storage is a flat # list in scada_report_schedules.json; cadence drives the date window the # scheduler computes at run time, so only the relative window is persisted. # --------------------------------------------------------------------------- SCADA_REPORT_CADENCES = ("last_week", "last_month", "custom") def _load_scada_report_schedules(): """Return the list of saved report schedules (empty list if none).""" if SCADA_REPORT_SCHEDULES_FILE.is_file(): try: saved = json.loads(SCADA_REPORT_SCHEDULES_FILE.read_text()) if isinstance(saved, list): return [s for s in saved if isinstance(s, dict)] except (json.JSONDecodeError, OSError): pass return [] def _save_scada_report_schedules(schedules): """Persist the report schedules list to disk.""" SCADA_REPORT_SCHEDULES_FILE.write_text(json.dumps(schedules, indent=2)) def _scada_schedule_from_body(body, existing=None): """Build/validate a schedule dict from a request body. Returns (schedule, error). ``existing`` supplies id/created_at when updating.""" name = (body.get("name") or "").strip() if not name: return None, "Schedule name is required." if len(name) > 120: return None, "Schedule name is too long (max 120 characters)." cadence = (body.get("cadence") or "last_week").strip() if cadence not in SCADA_REPORT_CADENCES: return None, f"Invalid cadence; expected one of {', '.join(SCADA_REPORT_CADENCES)}." custom_days = None if cadence == "custom": try: custom_days = int(body.get("custom_days")) except (TypeError, ValueError): return None, "custom_days must be an integer for a custom cadence." if custom_days < 1: return None, "custom_days must be at least 1." sensors = body.get("sensors") if sensors is None: sensors = [] if not isinstance(sensors, list): return None, "sensors must be a list." sensors = [s.strip() for s in sensors if isinstance(s, str) and s.strip()] now = _datetime_type.now().isoformat() schedule = { "id": (existing or {}).get("id") or str(uuid.uuid4()), "name": name, "cadence": cadence, "custom_days": custom_days, "sensors": sensors, "notify": bool(body.get("notify")), "external": bool(body.get("external")), "created_at": (existing or {}).get("created_at") or now, "updated_at": now, } return schedule, None @app.route("/scada/report-schedules") @scada_required def scada_report_schedules_list(): """Return all saved report schedules.""" return jsonify(schedules=_load_scada_report_schedules()) @app.route("/scada/report-schedules", methods=["POST"]) @scada_required def scada_report_schedules_create(): """Create a new report schedule.""" body = request.get_json(silent=True) or {} schedule, err = _scada_schedule_from_body(body) if err: return jsonify(error=err), 400 schedules = _load_scada_report_schedules() schedules.append(schedule) _save_scada_report_schedules(schedules) return jsonify(schedule=schedule), 201 @app.route("/scada/report-schedules/", methods=["PUT"]) @scada_required def scada_report_schedules_update(sched_id): """Update an existing report schedule by id.""" body = request.get_json(silent=True) or {} schedules = _load_scada_report_schedules() idx = next((i for i, s in enumerate(schedules) if s.get("id") == sched_id), None) if idx is None: return jsonify(error="Schedule not found."), 404 schedule, err = _scada_schedule_from_body(body, existing=schedules[idx]) if err: return jsonify(error=err), 400 schedules[idx] = schedule _save_scada_report_schedules(schedules) return jsonify(schedule=schedule) @app.route("/scada/report-schedules/", methods=["DELETE"]) @scada_required def scada_report_schedules_delete(sched_id): """Delete a report schedule by id.""" schedules = _load_scada_report_schedules() remaining = [s for s in schedules if s.get("id") != sched_id] if len(remaining) == len(schedules): return jsonify(error="Schedule not found."), 404 _save_scada_report_schedules(remaining) return jsonify(ok=True) # --------------------------------------------------------------------------- # SCADA report recipients (DVI-487) — two global email lists consumed by the # ST4 scheduler. Enforced recipients receive every scheduled report; External # recipients receive a report only when that schedule's `external` toggle is # on. File shape matches the scheduler's reader: {"recipients": [...]}. # --------------------------------------------------------------------------- SCADA_RECIPIENT_FILES = { "enforced": SCADA_ENFORCED_RECIPIENTS_FILE, "external": SCADA_EXTERNAL_RECIPIENTS_FILE, } def _load_scada_recipients(path): """Return the email list from a recipient file (flat list or {recipients}).""" if path.is_file(): try: data = json.loads(path.read_text()) except (json.JSONDecodeError, OSError): data = None if isinstance(data, dict): data = data.get("recipients", []) if isinstance(data, list): return [e for e in data if isinstance(e, str) and e.strip()] return [] def _save_scada_recipients(path, emails): """Persist a recipient list, de-duplicated and lowercased.""" seen, ordered = set(), [] for e in emails: if not isinstance(e, str): continue norm = e.strip().lower() if norm and norm not in seen: seen.add(norm) ordered.append(norm) path.write_text(json.dumps({"recipients": ordered}, indent=2)) @app.route("/scada/report-recipients/", methods=["GET"]) @scada_required def scada_report_recipients_get(list_name): """Return the enforced or external recipient list.""" path = SCADA_RECIPIENT_FILES.get(list_name) if path is None: return jsonify(error="Unknown recipient list."), 404 return jsonify(recipients=_load_scada_recipients(path)) @app.route("/scada/report-recipients/", methods=["POST"]) @scada_required def scada_report_recipients_post(list_name): """Add or remove an email in the enforced or external recipient list.""" path = SCADA_RECIPIENT_FILES.get(list_name) if path is None: return jsonify(error="Unknown recipient list."), 404 body = request.get_json(silent=True) or {} action = body.get("action", "") email = (body.get("email") or "").strip() if action not in ("add", "remove"): return jsonify(error="Invalid action."), 400 norm = email.lower() if action == "add" and ("@" not in norm or "." not in norm.split("@")[-1]): return jsonify(error="A valid email address is required."), 400 emails = _load_scada_recipients(path) if action == "add": if norm not in [e.lower() for e in emails]: emails.append(norm) else: emails = [e for e in emails if e.lower() != norm] try: _save_scada_recipients(path, emails) except OSError as exc: return jsonify(error=f"Failed to save: {exc}"), 500 return jsonify(ok=True, recipients=_load_scada_recipients(path)) # --------------------------------------------------------------------------- # SCADA notification delivery engine (DVI-468) # --------------------------------------------------------------------------- # Background collector that polls the SCADA mailbox on the configured # check_interval and fires the per-user emails that DVI-465 only persisted prefs # for: stale-data (no new reading within stale_after_minutes) and sensor # out-of-range. Recipient resolution + send plumbing live in notifications.py; # this is the wiring that consumes scada_config.json + per-user prefs. # Tracks which alert conditions have already been emailed so we fire once on the # transition into the condition and reset on recovery — never spam every poll. SCADA_ALERT_STATE_FILE = Path(__file__).resolve().parent / "scada_alert_state.json" def _scada_fetch_messages(top): """Fetch up to ``top`` recent SCADA mailbox messages. Returns (messages, error). Background-thread safe (returns plain data, not Flask responses). ``error`` is None on success or a short string describing the failure. """ try: app_client = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) token_result = app_client.acquire_token_for_client( scopes=["https://graph.microsoft.com/.default"]) token = token_result.get("access_token") if not token: return [], token_result.get("error_description", "no access_token") resp = _http_requests.get( f"https://graph.microsoft.com/v1.0/users/{SCADA_MAILBOX}/messages" f"?$top={int(top)}&$orderby=receivedDateTime desc" "&$select=subject,receivedDateTime,bodyPreview,body", headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, timeout=30, ) if resp.status_code != 200: return [], f"Graph API {resp.status_code}" return resp.json().get("value", []), None except Exception as exc: # network / parse errors — treat as transient return [], str(exc) def _scada_is_sensor_label(label): """Mirror the UI's _scadaIsSensorLabel: Temp/RH labels are sensors.""" return "Temp" in label or "RH" in label def _scada_label_enabled(config, label): """Mirror _scadaIsEnabled: enabled unless explicitly set False.""" enabled = config.get("enabled") or {} return enabled.get(label) is not False def _scada_reporting_disabled(config, label): """Mirror _scadaReportingDisabled: reporting[label] === false.""" reporting = config.get("reporting") or {} return reporting.get(label) is False def _scada_notify_enabled(config, label): """Per-sensor out-of-range notify flag (DVI-483): on unless set False.""" notify = config.get("notify") or {} return notify.get(label) is not False def _scada_value_out_of_range(rng, value): """True if numeric ``value`` falls outside the configured ``rng`` (low/high).""" if not isinstance(rng, dict): return False try: num = float(value) except (TypeError, ValueError): return False low = rng.get("low") high = rng.get("high") if low not in (None, "") and num < float(low): return True if high not in (None, "") and num > float(high): return True return False def _load_scada_alert_state(): """Return {stale: bool, oor: {label: bool}} persisted alert state.""" state = {"stale": False, "oor": {}} if SCADA_ALERT_STATE_FILE.is_file(): try: saved = json.loads(SCADA_ALERT_STATE_FILE.read_text()) if isinstance(saved, dict): state["stale"] = bool(saved.get("stale")) if isinstance(saved.get("oor"), dict): state["oor"] = {k: bool(v) for k, v in saved["oor"].items()} except (json.JSONDecodeError, OSError): pass return state def _save_scada_alert_state(state): try: SCADA_ALERT_STATE_FILE.write_text(json.dumps(state, indent=2)) except OSError: app.logger.exception("Failed to persist scada_alert_state.json") def _run_scada_collector(): """One collector cycle: poll the mailbox, fire stale/out-of-range emails. Emails fire only on the transition into a condition (stale onset, sensor crossing out of range) and the state resets on recovery, so a sustained condition is not re-emailed every poll. Returns a short summary dict. """ config = _load_scada_config() collector = config.get("collector") or dict(SCADA_COLLECTOR_DEFAULTS) stale_after = int(collector.get("stale_after_minutes", SCADA_COLLECTOR_DEFAULTS["stale_after_minutes"])) # Fetch a small window, not just the newest message: staleness is judged # against the most recent email of any type (any arrival means data is # flowing), but out-of-range checks must parse the latest Trigger email, # since "X-400 - Log" daily tables don't fit the Trigger snapshot parser # (DVI-538). messages, error = _scada_fetch_messages(25) if error: app.logger.warning("SCADA collector fetch failed: %s", error) return {"error": error} if not messages: return {"skipped": "no messages"} msg = messages[0] received_raw = msg.get("receivedDateTime", "") received_dt = None if received_raw: try: received_dt = _datetime_type.fromisoformat(received_raw.replace("Z", "+00:00")) except ValueError: received_dt = None state = _load_scada_alert_state() fired = {"stale": False, "oor": []} # --- Stale-data check (transition-guarded) --- age_minutes = None is_stale = False if received_dt is not None: now = _datetime_type.now(_dt_timezone.utc) age_minutes = int((now - received_dt).total_seconds() // 60) is_stale = age_minutes >= stale_after if is_stale and not state["stale"]: last_label = received_dt.strftime("%Y-%m-%d %H:%M UTC") if received_dt else None notifications.notify_scada( notifications.EVENT_SCADA_STALE, {"stale_after_minutes": stale_after, "age_minutes": age_minutes, "last_reading_time": last_label}) fired["stale"] = True state["stale"] = is_stale # --- Out-of-range check per enabled, reporting-on sensor --- trigger_msg = next( (m for m in messages if scada_subject_kind(m.get("subject")) == "trigger"), None) if trigger_msg is None: _save_scada_alert_state(state) return fired timestamp, readings = _parse_scada_body(_scada_message_body(trigger_msg)) ranges = config.get("ranges") or {} aliases = config.get("aliases") or {} seen_labels = set() for r in readings: label = r.get("label") if not label or not _scada_is_sensor_label(label): continue if not _scada_label_enabled(config, label) or _scada_reporting_disabled(config, label): continue if not _scada_notify_enabled(config, label): continue rng = ranges.get(label) if not isinstance(rng, dict): continue seen_labels.add(label) oor = _scada_value_out_of_range(rng, r.get("value")) prev = state["oor"].get(label, False) if oor and not prev: notifications.notify_scada( notifications.EVENT_SCADA_OUT_OF_RANGE, {"sensor": aliases.get(label) or label, "label": label, "value": r.get("value"), "low": rng.get("low"), "high": rng.get("high")}) fired["oor"].append(label) state["oor"][label] = oor _save_scada_alert_state(state) return fired # --------------------------------------------------------------------------- # OneDrive connector (DVI-810): File Browser tab + Winston app-only access # --------------------------------------------------------------------------- def _onedrive_graph_token() -> str | None: """Acquire an app-only Graph token scoped to the home (iCast) tenant.""" if not (_AZ_CLIENT_ID and _AZ_SECRET): return None try: cl = msal.ConfidentialClientApplication( _AZ_CLIENT_ID, authority=_AZ_AUTHORITY_HOME, client_credential=_AZ_SECRET) result = cl.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"]) token = result.get("access_token") if not token: app.logger.error("OneDrive: token request failed: %s", result.get("error_description", result)) return token except Exception: app.logger.exception("OneDrive: token acquisition error") return None def _onedrive_list_items(token: str, user_upn: str, item_id: str | None) -> list[dict] | None: """List OneDrive items — root if item_id is None, otherwise by item ID.""" if item_id: url = f"{_GRAPH_BASE}/users/{user_upn}/drive/items/{item_id}/children" else: url = f"{_GRAPH_BASE}/users/{user_upn}/drive/root/children" try: resp = _http_requests.get(url, headers={"Authorization": f"Bearer {token}"}, timeout=30) if resp.status_code != 200: app.logger.error("OneDrive list failed (%s) for %s item=%s: %s", resp.status_code, user_upn, item_id, resp.text[:300]) return None return resp.json().get("value", []) except Exception: app.logger.exception("OneDrive: list error for %s", user_upn) return None @app.route("/onedrive/ls") @login_required def onedrive_ls(): """List a OneDrive folder for the signed-in user using app-only Graph. Query params: item_id (optional) Drive item ID to list; defaults to root. Returns JSON {items: [{id, name, type, size, modified}]} or {error: ...}. """ user = session.get("user") or {} user_upn = user.get("email", "") if not user_upn: return jsonify(error="No user session"), 401 item_id = request.args.get("item_id") or None token = _onedrive_graph_token() if not token: return jsonify(error="OneDrive not available — Graph credentials not configured"), 503 raw = _onedrive_list_items(token, user_upn, item_id) if raw is None: return jsonify(error="Could not list OneDrive folder. Check that Files.ReadWrite.All is granted for this tenant."), 502 items = [] for it in raw: is_folder = "folder" in it items.append({ "id": it.get("id", ""), "name": it.get("name", ""), "type": "folder" if is_folder else "file", "size": it.get("size"), "modified": it.get("lastModifiedDateTime"), }) return jsonify(items=items) @app.route("/onedrive/download") @login_required def onedrive_download(): """Proxy-download a OneDrive file by item ID for the signed-in user. Query params: item_id Drive item ID (required). name Suggested filename for Content-Disposition (optional). """ user = session.get("user") or {} user_upn = user.get("email", "") if not user_upn: return jsonify(error="No user session"), 401 item_id = request.args.get("item_id", "").strip() if not item_id: return jsonify(error="item_id required"), 400 token = _onedrive_graph_token() if not token: return jsonify(error="OneDrive not available"), 503 # Fetch the item metadata to get the preauthenticated download URL. try: meta_resp = _http_requests.get( f"{_GRAPH_BASE}/users/{user_upn}/drive/items/{item_id}", headers={"Authorization": f"Bearer {token}"}, timeout=30, ) if meta_resp.status_code != 200: return jsonify(error="Item not found"), 404 meta = meta_resp.json() except Exception: app.logger.exception("OneDrive download: metadata fetch failed") return jsonify(error="OneDrive error"), 502 download_url = meta.get("@microsoft.graph.downloadUrl") if not download_url: return jsonify(error="No download URL — item may be a folder"), 400 # Stream the file content through to the client. filename = request.args.get("name") or meta.get("name", "download") content_type = (meta.get("file") or {}).get("mimeType", "application/octet-stream") try: file_resp = _http_requests.get(download_url, timeout=120, stream=True) if file_resp.status_code != 200: return jsonify(error="Download failed"), 502 def generate(): for chunk in file_resp.iter_content(chunk_size=65536): yield chunk from flask import Response, stream_with_context headers = { "Content-Disposition": f'attachment; filename="{filename}"', "Content-Type": content_type, } if "Content-Length" in file_resp.headers: headers["Content-Length"] = file_resp.headers["Content-Length"] return Response(stream_with_context(generate()), headers=headers) except Exception: app.logger.exception("OneDrive download: stream failed") return jsonify(error="Download error"), 502 # --------------------------------------------------------------------------- # OneDrive / SharePoint folder-browse endpoint (DVI-841) # Used by the quote-path picker (DVI-839 P3) to let users navigate their # personal OneDrive or the shared SharePoint library without leaving the app. # --------------------------------------------------------------------------- # Simple in-process TTL cache for the SharePoint drive ID (resolving it costs 2 # Graph round-trips and the ID is stable for the lifetime of the site). _sp_drive_cache: dict[str, tuple[str, float]] = {} # site_name → (drive_id, expire_ts) _SP_DRIVE_CACHE_TTL = 3600 # seconds def _sp_resolve_drive(token: str, site_name: str) -> str | None: """Return the default document-library drive ID for a SharePoint site. Caches the result for _SP_DRIVE_CACHE_TTL seconds to avoid redundant lookups. Requires Sites.Read.All (or Sites.ReadWrite.All) application permission. Returns None on any Graph error. """ import time as _time cached = _sp_drive_cache.get(site_name) if cached: drive_id, expire_ts = cached if _time.monotonic() < expire_ts: return drive_id hostname = _SP_HOSTNAME # "icastinc.sharepoint.com" # Step 1: resolve site ID via /sites/{hostname}:/sites/{site_name}: try: site_resp = _http_requests.get( f"{_GRAPH_BASE}/sites/{hostname}:/sites/{site_name}:", headers={"Authorization": f"Bearer {token}"}, timeout=30, ) if site_resp.status_code != 200: app.logger.error( "_sp_resolve_drive: site fetch failed (%s) for %s: %s", site_resp.status_code, site_name, site_resp.text[:300], ) return None site_id = site_resp.json().get("id", "") if not site_id: app.logger.error("_sp_resolve_drive: no site id in response for %s", site_name) return None except Exception: app.logger.exception("_sp_resolve_drive: site fetch error for %s", site_name) return None # Step 2: list drives and pick the default document library (Shared Documents). try: drives_resp = _http_requests.get( f"{_GRAPH_BASE}/sites/{site_id}/drives", headers={"Authorization": f"Bearer {token}"}, timeout=30, ) if drives_resp.status_code != 200: app.logger.error( "_sp_resolve_drive: drives fetch failed (%s) for site %s: %s", drives_resp.status_code, site_id, drives_resp.text[:300], ) return None drives = drives_resp.json().get("value", []) except Exception: app.logger.exception("_sp_resolve_drive: drives fetch error for site %s", site_id) return None # Prefer drive named "Documents" / "Shared Documents"; fall back to first documentLibrary. preferred_names = {"Documents", "Shared Documents", "General"} default_drive = None for d in drives: if d.get("driveType") == "documentLibrary": if d.get("name") in preferred_names: default_drive = d break if default_drive is None: default_drive = d if not default_drive: app.logger.error( "_sp_resolve_drive: no documentLibrary drive found for site %s (drives: %d)", site_id, len(drives), ) return None drive_id = default_drive.get("id", "") if drive_id: _sp_drive_cache[site_name] = (drive_id, _time.monotonic() + _SP_DRIVE_CACHE_TTL) return drive_id or None def _onedrive_browse_list(token: str, url: str, label: str) -> list[dict] | None: """Fetch a Graph /children URL and return the item list, or None on error. Selects only the fields needed by the browse response to reduce payload size. """ try: resp = _http_requests.get( url, headers={"Authorization": f"Bearer {token}"}, params={"$select": "id,name,size,folder,lastModifiedDateTime", "$top": 500}, timeout=30, ) if resp.status_code != 200: app.logger.error( "onedrive-browse: list failed (%s) for %s: %s", resp.status_code, label, resp.text[:300], ) return None return resp.json().get("value", []) except Exception: app.logger.exception("onedrive-browse: list error for %s", label) return None @app.route("/documents/onedrive-browse") @login_required def documents_onedrive_browse(): """Browse OneDrive/SharePoint folders for the quote-path picker (DVI-841). Query params: scope "personal" — signed-in user's OneDrive via /users/{upn}/drive "shared" — SharePoint site "InfrastructurePrecastInc" default library path Relative folder path from the drive root (default: root). Empty or "/" means root. Segments must not start/end with "/". Response JSON: { scope, path, breadcrumb:[{name,path}], folders:[{name,path,id}], files:[{name,path,id}] } Errors: 400 bad params, 502 Graph error, 503 credentials unavailable. Auth note (personal scope): uses app-only token reading /users/{upn}/drive, which requires Files.ReadWrite.All (application permission). If the tenant has not granted this, Graph returns 403. Delegated access (OBO flow) would be needed as an alternative — but that requires a separate OAuth redirect not currently wired up. """ user = session.get("user") or {} scope = request.args.get("scope", "personal").strip() if scope not in ("personal", "shared"): return jsonify(error="scope must be 'personal' or 'shared'"), 400 raw_path = request.args.get("path", "").strip().strip("/") token = _onedrive_graph_token() if not token: return jsonify( error="Graph credentials not configured — OneDrive/SharePoint browsing unavailable" ), 503 # Build breadcrumb from path segments. breadcrumb = [{"name": "Home", "path": ""}] if raw_path: parts = raw_path.split("/") for i, part in enumerate(parts): breadcrumb.append({"name": part, "path": "/".join(parts[: i + 1])}) if scope == "personal": user_upn = user.get("email", "") if not user_upn: return jsonify(error="No authenticated user UPN in session"), 401 if raw_path: url = f"{_GRAPH_BASE}/users/{user_upn}/drive/root:/{raw_path}:/children" else: url = f"{_GRAPH_BASE}/users/{user_upn}/drive/root/children" items = _onedrive_browse_list(token, url, f"personal:{user_upn}/{raw_path or 'root'}") if items is None: return jsonify( error=( "Could not list personal OneDrive folder. " "Ensure Files.ReadWrite.All (application) is granted for this tenant." ) ), 502 else: # scope == "shared" sp_site_name = "InfrastructurePrecastInc" drive_id = _sp_resolve_drive(token, sp_site_name) if drive_id is None: return jsonify( error=( "Could not resolve SharePoint site or drive. " "Ensure Sites.Read.All (or Sites.Selected) is granted for this tenant." ) ), 502 if raw_path: url = f"{_GRAPH_BASE}/drives/{drive_id}/root:/{raw_path}:/children" else: url = f"{_GRAPH_BASE}/drives/{drive_id}/root/children" items = _onedrive_browse_list(token, url, f"shared:{sp_site_name}/{raw_path or 'root'}") if items is None: return jsonify( error=( "Could not list SharePoint folder. " "Ensure Sites.Read.All (or Sites.Selected) is granted for this tenant." ) ), 502 folders = [] files = [] for it in items: name = it.get("name", "") entry = { "name": name, "path": f"{raw_path}/{name}" if raw_path else name, "id": it.get("id", ""), } if "folder" in it: folders.append(entry) else: files.append(entry) folders.sort(key=lambda x: x["name"].lower()) files.sort(key=lambda x: x["name"].lower()) return jsonify(scope=scope, path=raw_path, breadcrumb=breadcrumb, folders=folders, files=files) # --------------------------------------------------------------------------- # Vision tool (DVI-399 / DVI-408): enrollment, check-in, collectors, settings # --------------------------------------------------------------------------- # 14 agent-reported fields accepted in every check-in payload. _VISION_CHECKIN_FIELDS = ( "machine_id", "hostname", "logged_in_user", "os_version", "agent_version", "status", "capture_enabled", "last_capture_ts", "foreground_app", "idle_seconds", "cpu_pct", "screen_count", "agent_uptime_s", "ts", ) # Enrollment tokens expire after 60 minutes — a 15-minute window was too tight # when the first bootstrap attempt fails (e.g. ZIP download error) and the user # retries after troubleshooting. _VISION_ENROLL_TTL_S = 60 * 60 # vision_checkins is unbounded (one row per check-in). Keep raw history for this # many days; older rows are pruned on insert so the Activity timeline stays bounded. _VISION_CHECKIN_RETENTION_DAYS = 14 # Collector Phase-3 sparkline (DVI-427): online/offline history bucketed over # this window so each tile shows a recent uptime trend without a per-tile fetch. _VISION_SPARK_WINDOW_S = 6 * 3600 _VISION_SPARK_BUCKETS = 24 # Alert thresholds. A machine is "unexpectedly offline" once last_seen is older # than this (longer than the collector grid's online/offline staleness so we only # alert on sustained outages, not a single missed beat). _VISION_OFFLINE_ALERT_SECONDS = 15 * 60 # Vision alert log, separate from the WO idempotency notification_log.json. VISION_NOTIFICATION_LOG_FILE = Path(__file__).resolve().parent / "vision_notification_log.json" _vision_notif_lock = threading.Lock() def _vision_db(): """Open the Vision SQLite DB, creating tables on first use. Caller must close.""" conn = sqlite3.connect(str(VISION_DB_PATH), check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.executescript(""" CREATE TABLE IF NOT EXISTS vision_enrollment_tokens ( token TEXT PRIMARY KEY, machine_id TEXT, issued_at REAL NOT NULL, expires_at REAL NOT NULL, consumed_at REAL ); CREATE TABLE IF NOT EXISTS vision_agents ( machine_id TEXT PRIMARY KEY, hostname TEXT, agent_token TEXT UNIQUE NOT NULL, enrolled_at REAL NOT NULL, last_seen_at REAL, revoked INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'active' ); CREATE TABLE IF NOT EXISTS vision_kv ( key TEXT PRIMARY KEY, value TEXT, updated_at REAL ); CREATE TABLE IF NOT EXISTS vision_checkins ( id INTEGER PRIMARY KEY AUTOINCREMENT, machine_id TEXT NOT NULL, received_at REAL NOT NULL, hostname TEXT, logged_in_user TEXT, os_version TEXT, agent_version TEXT, status TEXT, capture_enabled INTEGER, last_capture_ts REAL, foreground_app TEXT, idle_seconds REAL, cpu_pct REAL, screen_count INTEGER, agent_uptime_s REAL, ts REAL, FOREIGN KEY (machine_id) REFERENCES vision_agents (machine_id) ); CREATE TABLE IF NOT EXISTS vision_commands ( id TEXT PRIMARY KEY, machine_id TEXT NOT NULL, action TEXT NOT NULL, params_json TEXT, created_at REAL NOT NULL, delivered_at REAL, acked_at REAL, acked_status TEXT, error_message TEXT, FOREIGN KEY (machine_id) REFERENCES vision_agents (machine_id) ); CREATE TABLE IF NOT EXISTS vision_agent_logs ( machine_id TEXT NOT NULL, file TEXT NOT NULL, content TEXT, received_at REAL NOT NULL, PRIMARY KEY (machine_id, file) ); CREATE INDEX IF NOT EXISTS idx_vision_checkins_machine_time ON vision_checkins (machine_id, received_at); """) _vision_migrate(conn) return conn # Enrollment status model (DVI-436). `status` is authoritative: # pending — self-enrolled (NULL-machine_id token or open-enroll secret); quarantined. # active — quick-enrolled (admin machine-bound token) or approved; surfaces normally. # revoked — token rejected; check-ins refused. # The legacy `revoked` INTEGER is kept in sync (1 iff status='revoked') so existing # `revoked = 0` queries continue to admit both pending and active rows. _VISION_AGENT_STATUSES = ("pending", "active", "revoked") def _vision_migrate(conn): """Idempotent schema upgrades for older Vision DBs (DVI-436, DVI-726).""" # vision_agents.status: add + backfill from the legacy revoked flag. agent_cols = {r["name"] for r in conn.execute("PRAGMA table_info(vision_agents)")} if "status" not in agent_cols: conn.execute( "ALTER TABLE vision_agents ADD COLUMN status TEXT NOT NULL DEFAULT 'active'" ) conn.execute("UPDATE vision_agents SET status = 'revoked' WHERE revoked = 1") conn.commit() # vision_enrollment_tokens.machine_id: drop the NOT NULL constraint so admins can # issue a token without a pre-known machine (agent self-supplies the id at register). tok_info = list(conn.execute("PRAGMA table_info(vision_enrollment_tokens)")) mid = next((r for r in tok_info if r["name"] == "machine_id"), None) if mid is not None and mid["notnull"] == 1: conn.executescript(""" CREATE TABLE vision_enrollment_tokens_new ( token TEXT PRIMARY KEY, machine_id TEXT, issued_at REAL NOT NULL, expires_at REAL NOT NULL, consumed_at REAL ); INSERT INTO vision_enrollment_tokens_new (token, machine_id, issued_at, expires_at, consumed_at) SELECT token, machine_id, issued_at, expires_at, consumed_at FROM vision_enrollment_tokens; DROP TABLE vision_enrollment_tokens; ALTER TABLE vision_enrollment_tokens_new RENAME TO vision_enrollment_tokens; """) conn.commit() # DVI-726: SSH public key fields. if "ssh_public_key" not in agent_cols: conn.execute("ALTER TABLE vision_agents ADD COLUMN ssh_public_key TEXT") conn.commit() if "ssh_key_written" not in agent_cols: conn.execute( "ALTER TABLE vision_agents ADD COLUMN ssh_key_written INTEGER NOT NULL DEFAULT 0" ) conn.commit() _VISION_OPEN_ENROLL_KEY = "open_enroll_secret" def _vision_write_ssh_key_to_rsync(public_key): """Append public_key to authorized_keys on the rsync server (DVI-726). Uses ssh to run a one-liner on the remote that appends the key only if it is not already present (idempotent). Returns True on success or a string error message on failure. Requires _VISION_RSYNC_MGMT_KEY to be set to a private key file that is pre-authorized for _VISION_RSYNC_USER@_VISION_RSYNC_HOST. """ if not _VISION_RSYNC_MGMT_KEY: return "VISION_RSYNC_MGMT_KEY not configured — cannot push key automatically." key_stripped = public_key.strip().replace("'", "'\\''") # Idempotent: append only if not already present. remote_cmd = ( f"mkdir -p ~/.ssh && chmod 700 ~/.ssh && " f"grep -qxF '{key_stripped}' ~/.ssh/authorized_keys 2>/dev/null || " f"echo '{key_stripped}' >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" ) cmd = [ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-p", str(_VISION_RSYNC_PORT), "-i", _VISION_RSYNC_MGMT_KEY, f"{_VISION_RSYNC_USER}@{_VISION_RSYNC_HOST}", remote_cmd, ] try: result = subprocess.run(cmd, capture_output=True, text=True, timeout=20) if result.returncode == 0: return True err = (result.stderr or result.stdout or "").strip() return err or f"ssh exited {result.returncode}" except subprocess.TimeoutExpired: return "SSH connection timed out." except FileNotFoundError: return "ssh not found on server PATH." except Exception as exc: # noqa: BLE001 return str(exc) def _vision_kv_get(conn, key): row = conn.execute("SELECT value FROM vision_kv WHERE key = ?", (key,)).fetchone() return row["value"] if row else None def _vision_kv_set(conn, key, value): conn.execute( "INSERT INTO vision_kv (key, value, updated_at) VALUES (?, ?, ?)" " ON CONFLICT(key) DO UPDATE SET value = excluded.value," " updated_at = excluded.updated_at", (key, value, time.time()), ) def _vision_get_open_enroll_secret(conn, create=True): """Return the current open-enroll shared secret, minting one on first use.""" secret = _vision_kv_get(conn, _VISION_OPEN_ENROLL_KEY) if not secret and create: secret = uuid.uuid4().hex _vision_kv_set(conn, _VISION_OPEN_ENROLL_KEY, secret) conn.commit() return secret def _vision_prune_checkins(conn): """Drop check-in rows older than the retention window. Cheap, index-backed.""" cutoff = time.time() - _VISION_CHECKIN_RETENTION_DAYS * 86400 conn.execute("DELETE FROM vision_checkins WHERE received_at < ?", (cutoff,)) def _load_vision_settings(): """Return Vision settings: {mock_data: bool}. Mock defaults on until first real check-in.""" defaults = {"mock_data": True} if VISION_SETTINGS_FILE.is_file(): try: saved = json.loads(VISION_SETTINGS_FILE.read_text()) if isinstance(saved.get("mock_data"), bool): defaults["mock_data"] = saved["mock_data"] except (json.JSONDecodeError, OSError): pass return defaults def _save_vision_settings(settings): VISION_SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) _vision_names_lock = threading.Lock() def _load_vision_agent_names(): """Return {machine_id: friendly_name} display-name overrides (DVI-682).""" if not VISION_AGENT_NAMES_FILE.is_file(): return {} try: data = json.loads(VISION_AGENT_NAMES_FILE.read_text()) return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {} except (json.JSONDecodeError, OSError): return {} def _save_vision_agent_names(names): VISION_AGENT_NAMES_FILE.write_text(json.dumps(names, indent=2)) # --- Capture policy (DVI-427) -------------------------------------------- # Default policy applied to every machine; per-machine overrides live under # "machines". Delivered to agents via the existing check-in response config_delta. _VISION_POLICY_DEFAULTS = { "capture_enabled": False, # capture is off until explicitly enabled "capture_mode": "screenshot", # screenshot | video (DVI-727 hybrid backend) "capture_fps": 1.0, # video frame rate; ignored in screenshot mode "capture_rate_s": 60, # seconds between frames (screenshot mode) "idle_threshold_s": 300, # pause capture after this idle window "exclusions": [], # app names / URL substrings never captured "schedule": {"days": [1, 2, 3, 4, 5], "start": "08:00", "end": "17:00"}, "retention_days": 30, # surfaced for the footage store's purge job } _VISION_CAPTURE_MODES = ("screenshot", "video") def _load_vision_policy(): """Return {default: {...}, machines: {machine_id: {...overrides}}}.""" base = {"default": dict(_VISION_POLICY_DEFAULTS), "machines": {}} if VISION_POLICY_FILE.is_file(): try: saved = json.loads(VISION_POLICY_FILE.read_text()) if isinstance(saved, dict): if isinstance(saved.get("default"), dict): base["default"].update(_vision_clean_policy(saved["default"])) if isinstance(saved.get("machines"), dict): for mid, ov in saved["machines"].items(): if isinstance(ov, dict): base["machines"][mid] = _vision_clean_policy(ov) except (json.JSONDecodeError, OSError): pass return base def _vision_clean_policy(raw): """Coerce a policy dict to known fields/types; drop anything unrecognized.""" out = {} if "capture_enabled" in raw: out["capture_enabled"] = bool(raw["capture_enabled"]) if "capture_mode" in raw: mode = str(raw["capture_mode"]).strip().lower() if mode in _VISION_CAPTURE_MODES: out["capture_mode"] = mode if "capture_fps" in raw: try: fps = float(raw["capture_fps"]) if fps > 0: # Clamp to a sane window: 0.1 fps (1 frame / 10s) up to 30 fps. out["capture_fps"] = round(min(30.0, max(0.1, fps)), 3) except (TypeError, ValueError): pass for k in ("capture_rate_s", "idle_threshold_s", "retention_days"): if k in raw: try: out[k] = max(0, int(raw[k])) except (TypeError, ValueError): pass if isinstance(raw.get("exclusions"), list): out["exclusions"] = [str(x).strip() for x in raw["exclusions"] if str(x).strip()] sched = raw.get("schedule") if isinstance(sched, dict): clean = {} if isinstance(sched.get("days"), list): clean["days"] = sorted({int(d) for d in sched["days"] if isinstance(d, (int, float)) and 0 <= int(d) <= 6}) for k in ("start", "end"): v = sched.get(k) if isinstance(v, str) and re.match(r"^\d{1,2}:\d{2}$", v.strip()): clean[k] = v.strip() out["schedule"] = clean return out def _save_vision_policy(policy): VISION_POLICY_FILE.write_text(json.dumps(policy, indent=2)) def _vision_effective_policy(machine_id, policy=None): """Merge default + per-machine override into the policy the agent receives.""" policy = policy or _load_vision_policy() eff = dict(_VISION_POLICY_DEFAULTS) eff.update(policy.get("default") or {}) override = (policy.get("machines") or {}).get(machine_id) if isinstance(override, dict): eff.update(override) return eff _vision_footage_purge_at = 0.0 _VISION_FOOTAGE_PURGE_INTERVAL_S = 3600 # at most one purge sweep per hour def _vision_purge_footage(now=None): """Delete footage older than each machine's effective retention_days. Covers screenshot frames, video segments (.mp4), and their thumbnail/poster companions alike — anything in the machine's footage dir whose age exceeds its retention window. Throttled to one sweep/hour so the check-in path stays cheap. retention_days <= 0 disables purging for that machine.""" global _vision_footage_purge_at now = now or time.time() if now - _vision_footage_purge_at < _VISION_FOOTAGE_PURGE_INTERVAL_S: return _vision_footage_purge_at = now if not VISION_FOOTAGE_DIR.is_dir(): return policy = _load_vision_policy() for mdir in VISION_FOOTAGE_DIR.iterdir(): if not mdir.is_dir(): continue days = _vision_effective_policy(mdir.name, policy).get("retention_days") or 0 if days <= 0: continue cutoff = now - days * 86400 for p in mdir.iterdir(): try: if p.is_file() and _vision_frame_ts(p) < cutoff: p.unlink() except OSError: continue # --- Captured footage store (D3) ----------------------------------------- # Frame ids and machine ids are constrained to this charset so a request can # never traverse outside the configured footage root. _VISION_FRAME_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") _VISION_FRAME_EXTS = (".jpg", ".jpeg", ".png", ".webp") # Segmented H.264 footage (DVI-727 video backend). A segment .mp4 ships # with poster companions (.jpg full + .thumb.jpg thumb) extracted # agent-side, so the existing image-serving path renders the tile/poster. _VISION_VIDEO_EXTS = (".mp4",) def _vision_footage_machine_dir(machine_id): return VISION_FOOTAGE_DIR / machine_id def _vision_machine_active(machine_id): """True only for an enrolled, approved (status='active') machine. Pending and revoked machines are quarantined from footage/recordings (DVI-436).""" conn = _vision_db() try: row = conn.execute( "SELECT status FROM vision_agents WHERE machine_id = ?", (machine_id,) ).fetchone() finally: conn.close() return bool(row) and row["status"] == "active" def _vision_frame_ts(path): """Frame timestamp: epoch-second filename stem if present, else mtime.""" try: stem_ts = float(path.stem) if stem_ts > 1_000_000_000: return stem_ts except ValueError: pass try: return path.stat().st_mtime except OSError: return 0.0 class FootageStore: """Backend-agnostic access to captured Vision footage (DVI-743 Phase 1). Serving endpoints address footage only by (machine_id, frame_id), so the storage backend is swappable behind this seam. Callers enforce the `vision_access_required` + `_vision_machine_active` quarantine *before* invoking any method here — the store never re-checks access. Phase 2 adds an on-demand remote backend implementing this same interface. """ def list_frames(self, machine_id, frm=None, to=None, limit=500): """Footage entries for a machine, newest first (frame + video dicts).""" raise NotImplementedError def frame_bytes(self, machine_id, frame_id, size): """Flask response serving the frame image (size 'thumb'|'full'), or None if the frame is absent.""" raise NotImplementedError def segment_stream(self, machine_id, frame_id, rng=None): """Range-capable Flask response for the H.264 segment, or None if absent. The local backend defers Range handling to Flask's conditional send_file (which reads the request's Range header); `rng` is the forward-compat seam for a remote backend that must forward the range.""" raise NotImplementedError def read_bytes(self, machine_id, frame_id, kind): """Raw footage bytes for Phase-4 Assistant ingestion (DVI-751). Unlike frame_bytes/segment_stream (which return Flask responses for the HTTP serving path), this returns ``(data: bytes, content_type: str)`` or None, so a future Assistant/Claude integration can build image blocks or upload via the Files API without round-tripping through HTTP. `kind` selects the representation: "thumb"|"full" -> frame image (a video item's poster keyframe for "full"), "segment" -> raw H.264 .mp4.""" raise NotImplementedError class LocalFootageStore(FootageStore): """Footage served from the local mirror directory (VISION_FOOTAGE_DIR//...). This is the pre-DVI-743 behavior, moved behind the FootageStore seam with no functional change.""" def _machine_dir(self, machine_id): return _vision_footage_machine_dir(machine_id) def list_frames(self, machine_id, frm=None, to=None, limit=500): """Real footage-store entries for a machine, newest first. Handles both screenshot frames (.jpg/.png/…) and video segments (.mp4 + poster), DVI-727. Thumbnails (.thumb.) are companions, not standalone entries. A poster image that shares a stem with a .mp4 segment is surfaced through the video entry (as its thumb/full poster), not as its own frame.""" d = self._machine_dir(machine_id) if not d.is_dir(): return [] video_stems = set() files = [] for p in sorted(d.iterdir()): if not p.is_file() or ".thumb." in p.name: continue ext = p.suffix.lower() if ext in _VISION_VIDEO_EXTS: video_stems.add(p.stem) files.append((p, "video")) elif ext in _VISION_FRAME_EXTS: files.append((p, "image")) out = [] for p, kind in files: # Skip poster images — they belong to their video segment's entry. if kind == "image" and p.stem in video_stems: continue ts = _vision_frame_ts(p) if frm is not None and ts < frm: continue if to is not None and ts > to: continue entry = { "frame_id": p.stem, "ts": int(ts), "kind": kind, "thumb_url": f"/vision/frames/{machine_id}/{p.stem}?size=thumb", "full_url": f"/vision/frames/{machine_id}/{p.stem}?size=full", } if kind == "video": entry["video_url"] = f"/vision/segments/{machine_id}/{p.stem}" out.append(entry) out.sort(key=lambda f: f["ts"], reverse=True) return out[:limit] def frame_bytes(self, machine_id, frame_id, size): d = self._machine_dir(machine_id) candidates = [] if size == "thumb": candidates += sorted(d.glob(frame_id + ".thumb.*")) candidates += [p for p in sorted(d.glob(frame_id + ".*")) if ".thumb." not in p.name] for p in candidates: if p.is_file() and p.suffix.lower() in _VISION_FRAME_EXTS: return send_file(str(p)) return None def segment_stream(self, machine_id, frame_id, rng=None): d = self._machine_dir(machine_id) for p in sorted(d.glob(frame_id + ".*")): if p.is_file() and p.suffix.lower() in _VISION_VIDEO_EXTS: return send_file(str(p), conditional=True) return None def read_bytes(self, machine_id, frame_id, kind): d = self._machine_dir(machine_id) if kind == "segment": for p in sorted(d.glob(frame_id + ".*")): if p.is_file() and p.suffix.lower() in _VISION_VIDEO_EXTS: return p.read_bytes(), _VISION_CONTENT_TYPES.get( p.suffix.lower(), "video/mp4") return None candidates = [] if kind == "thumb": candidates += sorted(d.glob(frame_id + ".thumb.*")) candidates += [p for p in sorted(d.glob(frame_id + ".*")) if ".thumb." not in p.name] for p in candidates: if p.is_file() and p.suffix.lower() in _VISION_FRAME_EXTS: return p.read_bytes(), _VISION_CONTENT_TYPES.get( p.suffix.lower(), "application/octet-stream") return None _VISION_CONTENT_TYPES = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", ".mp4": "video/mp4", } class _FootageLRUCache: """Bounded on-disk LRU hot-cache for remotely-fetched footage (DVI-748, D3=ii). Holds recently-viewed thumbs/frames/segments only — never a full mirror. Entries are evicted oldest-first once the total cached size exceeds ``max_bytes`` or an entry is older than ``ttl_s``. Files are stored under ``root`` with their original extension so callers can serve them straight through send_file (correct Content-Type + native HTTP Range for video). """ def __init__(self, root, max_bytes, ttl_s): self._root = Path(root) self._max_bytes = max(int(max_bytes), 0) self._ttl = max(int(ttl_s), 0) self._lock = threading.Lock() # key -> (path, size); insertion order == recency (move_to_end on hit). self._entries = _OrderedDict_type() self._mtimes = {} try: self._root.mkdir(parents=True, exist_ok=True) except OSError: pass def _safe_name(self, key): # Flatten machine_id/filename into one cache filename; key chars are # already restricted by _VISION_FRAME_ID_RE + a fixed extension set. return key.replace("/", "__") def get(self, key): now = time.time() with self._lock: entry = self._entries.get(key) if entry is None: return None path, _size = entry if self._ttl and now - self._mtimes.get(key, 0) > self._ttl: self._drop(key) return None if not path.exists(): self._drop(key) return None self._entries.move_to_end(key) return path def put(self, key, data): path = self._root / self._safe_name(key) try: tmp = path.with_suffix(path.suffix + ".part") tmp.write_bytes(data) tmp.replace(path) except OSError as exc: app.logger.warning("Footage cache write failed for %s: %s", key, exc) return None with self._lock: if key in self._entries: self._drop(key) self._entries[key] = (path, len(data)) self._mtimes[key] = time.time() self._evict_locked() return path def _drop(self, key): entry = self._entries.pop(key, None) self._mtimes.pop(key, None) if entry: try: entry[0].unlink() except OSError: pass def _evict_locked(self): # Never evict the just-added (newest) entry, even if it alone exceeds the # cap — that keeps an oversized item servable until the next put. total = sum(sz for _p, sz in self._entries.values()) while total > self._max_bytes and len(self._entries) > 1: old_key, (old_path, old_size) = next(iter(self._entries.items())) self._entries.pop(old_key, None) self._mtimes.pop(old_key, None) try: old_path.unlink() except OSError: pass total -= old_size class _SFTPPool: """Tiny thread-safe pool of paramiko SFTP connections (DVI-748). Avoids a per-request SSH handshake. Connections are validated cheaply on checkout; dead ones are discarded and replaced. paramiko is imported lazily so the local backend never pays for it. """ def __init__(self, host, port, user, key_path, max_size): self._host, self._port, self._user = host, port, user self._key_path = key_path self._max_size = max(int(max_size), 1) self._idle = [] # list[(transport_owner_client, sftp)] self._lock = threading.Lock() def _connect(self): import paramiko # lazy: only the remote backend needs it client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect( hostname=self._host, port=self._port, username=self._user, key_filename=self._key_path, timeout=15, banner_timeout=15, auth_timeout=15, look_for_keys=False, allow_agent=False, ) sftp = client.open_sftp() sftp.get_channel().settimeout(30) return client, sftp @_contextmanager_type def connection(self): client = sftp = None with self._lock: if self._idle: client, sftp = self._idle.pop() if sftp is not None: try: sftp.stat(".") # liveness probe except Exception: try: client.close() except Exception: pass client = sftp = None if sftp is None: client, sftp = self._connect() try: yield sftp except Exception: try: client.close() except Exception: pass raise else: with self._lock: if len(self._idle) < self._max_size: self._idle.append((client, sftp)) return try: client.close() except Exception: pass class RemoteSSHFootageStore(FootageStore): """Footage served on demand from the rsync storage server over SFTP (DVI-748 Phase 2). No full local copy — a bounded LRU caches only the hot set. Remote layout mirrors local: ``//`` (+ ``.thumb.`` companions, ``.mp4`` segments).""" def __init__(self): self._base = VISION_FOOTAGE_REMOTE_BASE.rstrip("/") self._pool = _SFTPPool( _VISION_RSYNC_HOST, _VISION_RSYNC_PORT, _VISION_RSYNC_USER, VISION_FOOTAGE_SSH_KEY, VISION_FOOTAGE_SSH_POOL_SIZE, ) self._cache = _FootageLRUCache( VISION_FOOTAGE_CACHE_DIR, VISION_FOOTAGE_CACHE_MAX_BYTES, VISION_FOOTAGE_CACHE_TTL, ) self._listing_ttl = max(VISION_FOOTAGE_LISTING_TTL, 0) self._listings = {} # machine_id -> (expires_at, [(name, mtime), ...]) self._listing_lock = threading.Lock() def _remote_dir(self, machine_id): return f"{self._base}/{machine_id}" def _list_remote(self, machine_id): """[(filename, mtime)] for a machine dir, briefly cached. [] on outage or a missing/empty dir (clean UI, never a crash).""" now = time.time() with self._listing_lock: cached = self._listings.get(machine_id) if cached and cached[0] > now: return cached[1] try: with self._pool.connection() as sftp: attrs = sftp.listdir_attr(self._remote_dir(machine_id)) files = [(a.filename, float(getattr(a, "st_mtime", 0) or 0)) for a in attrs] except FileNotFoundError: files = [] except Exception as exc: app.logger.warning("Remote footage listing failed for %s: %s", machine_id, exc) return [] if self._listing_ttl: with self._listing_lock: self._listings[machine_id] = (now + self._listing_ttl, files) return files @staticmethod def _name_ts(name, mtime): stem = name.rsplit(".", 1)[0] # A thumb companion's stem is ".thumb"; use the frame_id part. if stem.endswith(".thumb"): stem = stem[: -len(".thumb")] try: ts = float(stem) if ts > 1_000_000_000: return ts except ValueError: pass return mtime def list_frames(self, machine_id, frm=None, to=None, limit=500): files = self._list_remote(machine_id) video_stems = set() items = [] # (name, ext, kind, ts) for name, mtime in files: if ".thumb." in name: continue dot = name.rfind(".") if dot < 0: continue ext = name[dot:].lower() stem = name[:dot] if ext in _VISION_VIDEO_EXTS: video_stems.add(stem) items.append((name, ext, stem, "video", self._name_ts(name, mtime))) elif ext in _VISION_FRAME_EXTS: items.append((name, ext, stem, "image", self._name_ts(name, mtime))) out = [] for name, ext, stem, kind, ts in items: if kind == "image" and stem in video_stems: continue # poster belongs to its video entry if frm is not None and ts < frm: continue if to is not None and ts > to: continue entry = { "frame_id": stem, "ts": int(ts), "kind": kind, "thumb_url": f"/vision/frames/{machine_id}/{stem}?size=thumb", "full_url": f"/vision/frames/{machine_id}/{stem}?size=full", } if kind == "video": entry["video_url"] = f"/vision/segments/{machine_id}/{stem}" out.append(entry) out.sort(key=lambda f: f["ts"], reverse=True) return out[:limit] def _resolve(self, machine_id, frame_id, exts, prefer_thumb=False): """Pick the best matching remote filename for a frame, or None.""" files = [n for n, _m in self._list_remote(machine_id)] candidates = [] if prefer_thumb: candidates += sorted( n for n in files if n.startswith(frame_id + ".thumb.") and n[n.rfind("."):].lower() in exts ) candidates += sorted( n for n in files if ".thumb." not in n and n.startswith(frame_id + ".") and n[n.rfind("."):].lower() in exts ) return candidates[0] if candidates else None def _fetch(self, machine_id, name): """Return a local cache path for a remote file, fetching via SFTP on a miss. None on outage/missing file.""" cache_key = f"{machine_id}/{name}" hit = self._cache.get(cache_key) if hit is not None: return hit remote_path = f"{self._remote_dir(machine_id)}/{name}" try: with self._pool.connection() as sftp: with sftp.open(remote_path, "rb") as fh: fh.prefetch() data = fh.read() except FileNotFoundError: return None except Exception as exc: app.logger.warning("Remote footage fetch failed for %s: %s", remote_path, exc) return None return self._cache.put(cache_key, data) def frame_bytes(self, machine_id, frame_id, size): name = self._resolve(machine_id, frame_id, _VISION_FRAME_EXTS, prefer_thumb=(size == "thumb")) if not name: return None path = self._fetch(machine_id, name) if path is None: return None ctype = _VISION_CONTENT_TYPES.get(path.suffix.lower()) return send_file(str(path), mimetype=ctype) def segment_stream(self, machine_id, frame_id, rng=None): name = self._resolve(machine_id, frame_id, _VISION_VIDEO_EXTS) if not name: return None path = self._fetch(machine_id, name) if path is None: return None # Whole segment is cached locally (H.264 segments are small, ~10s at low # bitrate), so Flask's conditional send_file serves HTTP Range / 206 for # scrubbing natively off the cache file. return send_file(str(path), mimetype="video/mp4", conditional=True) def read_bytes(self, machine_id, frame_id, kind): if kind == "segment": name = self._resolve(machine_id, frame_id, _VISION_VIDEO_EXTS) else: name = self._resolve(machine_id, frame_id, _VISION_FRAME_EXTS, prefer_thumb=(kind == "thumb")) if not name: return None path = self._fetch(machine_id, name) if path is None: return None default_ct = "video/mp4" if kind == "segment" else "application/octet-stream" ctype = _VISION_CONTENT_TYPES.get(path.suffix.lower(), default_ct) return path.read_bytes(), ctype def _make_footage_store(): """Select the footage backend from VISION_FOOTAGE_BACKEND (DVI-743). "remote" (DVI-748) streams on demand over SFTP; anything else (incl. an unknown value) falls back to the local-disk mirror so a misconfigured env never takes footage offline. A failure constructing the remote backend also degrades to local rather than crashing startup.""" if VISION_FOOTAGE_BACKEND == "remote": try: store = RemoteSSHFootageStore() app.logger.info( "Vision footage backend = remote SFTP (%s@%s:%s/%s)", _VISION_RSYNC_USER, _VISION_RSYNC_HOST, _VISION_RSYNC_PORT, VISION_FOOTAGE_REMOTE_BASE, ) return store except Exception as exc: app.logger.error( "Remote footage backend unavailable (%s); falling back to local.", exc ) return LocalFootageStore() if VISION_FOOTAGE_BACKEND not in ("", "local"): app.logger.warning( "VISION_FOOTAGE_BACKEND=%r is not recognized; using local backend.", VISION_FOOTAGE_BACKEND, ) return LocalFootageStore() _footage_store = _make_footage_store() def footage_bytes(machine_id, frame_id, kind="full"): """Phase-4 Assistant-ingestion resolver (DVI-751 — groundwork, NOT wired). The single server-side seam mapping a canonical Recordings item (machine_id, frame_id) to raw footage bytes for a future Assistant/Claude "analyze this capture" action. Returns ``(data: bytes, content_type: str)`` or None. Backed by the active FootageStore, so it behaves identically for the local-mirror and remote-SFTP backends. Ingestion mapping (the contract Vision Phase 4 will build on): - "full" -> full frame image. A screenshot item maps 1:1 to a Claude image block (base64). A video item yields its poster / keyframe image, the cheap representative still for a clip. - "thumb" -> small frame image (low-token preview / triage). - "segment"-> raw H.264 .mp4 bytes. The caller samples keyframes (or uploads via the Files API) — frame extraction is future Phase-4 work, deliberately out of scope here. Access: callers MUST enforce vision_access_required + _vision_machine_active before invoking; this resolver never re-checks access (same rule as the FootageStore serving methods). NOT YET WIRED: no Assistant tool registration, no Claude call, no UI, no endpoint. This locks the contract so the streaming refactor (DVI-743) is a clean base for Vision Phase 4.""" return _footage_store.read_bytes(machine_id, frame_id, kind) def _vision_mock_frames(machine_id, frm=None, to=None, count=24): """Synthetic frame list so Recordings/thumbnails are demonstrable pre-capture.""" now = int(time.time()) out = [] for i in range(count): ts = now - i * 900 # one mock frame every 15 minutes if frm is not None and ts < frm: continue if to is not None and ts > to: continue fid = f"mock-{ts}" out.append({ "frame_id": fid, "ts": ts, "kind": "image", "thumb_url": f"/vision/frames/{machine_id}/{fid}?size=thumb", "full_url": f"/vision/frames/{machine_id}/{fid}?size=full", }) return out def _vision_latest_frame(machine_id, mock): frames = (_vision_mock_frames(machine_id, count=1) if mock else _footage_store.list_frames(machine_id, limit=1)) return frames[0] if frames else None def _vision_mock_frame_svg(machine_id, frame_id, size): """A self-describing placeholder image for mock footage (no binary deps).""" w, h = (240, 150) if size == "thumb" else (960, 600) try: ts = int(frame_id.split("-", 1)[1]) when = _datetime_type.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") except (ValueError, IndexError): when = frame_id fs1 = 16 if size == "thumb" else 34 fs2 = 11 if size == "thumb" else 20 return ( f'' f'' f'' f'{_h(machine_id)}' f'{_h(when)}' f'mock frame' f'' ) def _h(s): """Minimal XML/HTML text escape for SVG bodies.""" return (str(s).replace("&", "&").replace("<", "<") .replace(">", ">").replace('"', """)) def _vision_mock_collectors(): """Sample machine tiles so the Collector UI is demonstrable pre-agent. Includes Phase-3 fields (sparkline/uptime/restarts/thumbnail).""" now = int(time.time()) full = [1] * _VISION_SPARK_BUCKETS gappy = [1] * _VISION_SPARK_BUCKETS for i in (3, 4, 11, 18): gappy[i] = 0 offline_tail = [1] * (_VISION_SPARK_BUCKETS - 6) + [0] * 6 return [ {"machine_id": "mock-1", "hostname": "ICAST-CNC-01", "logged_in_user": "jdoe", "agent_version": "0.1.0", "last_seen": now - 20, "online": True, "status": "active", "capture_enabled": True, "foreground_app": "Mastercam", "idle_seconds": 5, "cpu_pct": 37, "agent_uptime_s": 86400 * 3 + 1200, "restarts": 0, "sparkline": full, "thumb_url": f"/vision/frames/mock-1/mock-{now}?size=thumb", "full_url": f"/vision/frames/mock-1/mock-{now}?size=full"}, {"machine_id": "mock-2", "hostname": "ICAST-PRESS-02", "logged_in_user": "msmith", "agent_version": "0.1.0", "last_seen": now - 95, "online": True, "status": "idle", "capture_enabled": True, "foreground_app": "Outlook", "idle_seconds": 840, "cpu_pct": 8, "agent_uptime_s": 7200, "restarts": 2, "sparkline": gappy, "thumb_url": f"/vision/frames/mock-2/mock-{now}?size=thumb", "full_url": f"/vision/frames/mock-2/mock-{now}?size=full"}, {"machine_id": "mock-3", "hostname": "ICAST-WELD-03", "logged_in_user": "—", "agent_version": "0.1.0", "last_seen": now - 4200, "online": False, "status": "error", "capture_enabled": False, "foreground_app": None, "idle_seconds": None, "cpu_pct": None, "agent_uptime_s": None, "restarts": 1, "sparkline": offline_tail, "thumb_url": None, "full_url": None}, ] @app.route("/vision/agents/enroll", methods=["POST"]) @togen_admin_required def vision_agents_enroll(): """Issue a short-lived enrollment token. machine_id is optional (DVI-436): supplying it pre-binds the token to a known box → quick-enroll, agent lands `active`. Omitting it stores machine_id NULL → the agent self-supplies a generated id at register time and lands `pending` (admin approval required). """ body = request.get_json(silent=True) or {} machine_id = (body.get("machine_id") or "").strip() or None token = str(uuid.uuid4()) now = time.time() conn = _vision_db() try: conn.execute( "INSERT INTO vision_enrollment_tokens (token, machine_id, issued_at, expires_at)" " VALUES (?, ?, ?, ?)", (token, machine_id, now, now + _VISION_ENROLL_TTL_S), ) conn.commit() finally: conn.close() record_history( "vision-agents", "enroll-token-issued", actor=(session.get("user") or {}).get("name"), summary=f"Enrollment token issued for {machine_id or 'self-enroll'}", ref=machine_id or "", ) return jsonify(enrollment_token=token, machine_id=machine_id, mode="quick" if machine_id else "self") @app.route("/vision/agents/register", methods=["POST"]) def vision_agent_register(): """Machine presents a credential; receives a permanent agent_token. Three trust paths (DVI-436), all via the `enrollment_token` field so the existing agent needs no change: - machine-bound token → effective machine_id from the token, status active. - NULL-machine token → agent self-supplies machine_id, status pending. - open-enroll secret → agent self-supplies machine_id, status pending. The open-enroll secret may also be passed explicitly as `open_enroll_secret`. """ body = request.get_json(silent=True) or {} auth = request.headers.get("Authorization", "") credential = (body.get("enrollment_token") or (auth[7:].strip() if auth.startswith("Bearer ") else "")) open_secret_body = (body.get("open_enroll_secret") or "").strip() machine_id_body = (body.get("machine_id") or "").strip() hostname = (body.get("hostname") or "").strip() or None ssh_public_key = (body.get("ssh_public_key") or "").strip() or None if not credential and not open_secret_body: return jsonify(error="enrollment_token or open_enroll_secret required."), 400 now = time.time() conn = _vision_db() try: open_secret = _vision_get_open_enroll_secret(conn, create=False) via_open = bool(open_secret) and ( open_secret_body == open_secret or credential == open_secret ) if via_open: # Self-enroll via shared secret → pending, agent-supplied machine_id. if not machine_id_body: return jsonify(error="machine_id required for self-enroll."), 400 effective_machine_id, status = machine_id_body, "pending" else: row = conn.execute( "SELECT * FROM vision_enrollment_tokens WHERE token = ?", (credential,) ).fetchone() if not row: return jsonify(error="Invalid enrollment_token."), 401 if row["consumed_at"] is not None: return jsonify(error="enrollment_token already used."), 401 if now > row["expires_at"]: return jsonify(error="enrollment_token expired."), 401 if row["machine_id"]: # Machine-bound quick-enroll → active. if machine_id_body and machine_id_body != row["machine_id"]: return jsonify(error="machine_id does not match enrollment."), 403 effective_machine_id, status = row["machine_id"], "active" else: # NULL-machine token → agent self-supplies id, lands pending. if not machine_id_body: return jsonify(error="machine_id required for self-enroll."), 400 effective_machine_id, status = machine_id_body, "pending" conn.execute( "UPDATE vision_enrollment_tokens SET consumed_at = ? WHERE token = ?", (now, credential), ) existing = conn.execute( "SELECT agent_token, status, revoked FROM vision_agents WHERE machine_id = ?", (effective_machine_id,), ).fetchone() if existing and not existing["revoked"]: agent_token = existing["agent_token"] status = existing["status"] # re-register keeps prior disposition if ssh_public_key: conn.execute( "UPDATE vision_agents SET ssh_public_key = ? WHERE machine_id = ?", (ssh_public_key, effective_machine_id), ) elif existing: # Revoked machine re-enrolling with a fresh valid token: treat as a new # enrollment so an admin can re-approve. Issue a new agent_token (the old # one was revoked) and clear the revoked flag with the freshly computed status. agent_token = str(uuid.uuid4()) conn.execute( "UPDATE vision_agents SET agent_token = ?, hostname = COALESCE(?, hostname)," " enrolled_at = ?, status = ?, revoked = ?," " ssh_public_key = COALESCE(?, ssh_public_key) WHERE machine_id = ?", (agent_token, hostname, now, status, 1 if status == "revoked" else 0, ssh_public_key, effective_machine_id), ) else: agent_token = str(uuid.uuid4()) conn.execute( "INSERT INTO vision_agents" " (machine_id, hostname, agent_token, enrolled_at, status, revoked, ssh_public_key)" " VALUES (?, ?, ?, ?, ?, ?, ?)", (effective_machine_id, hostname, agent_token, now, status, 1 if status == "revoked" else 0, ssh_public_key), ) conn.commit() finally: conn.close() record_history( "vision-agents", "registered", actor=hostname or effective_machine_id, summary=f"Agent {effective_machine_id} registered (status: {status})", ref=effective_machine_id, ) return jsonify(agent_token=agent_token, machine_id=effective_machine_id, status=status) @app.route("/vision/agents/ssh-key", methods=["POST", "PUT"]) def vision_agent_ssh_key(): """Agent reports its SSH public key for rsync authorization (DVI-726). Bearer auth via existing agent_token. Stores the key so the admin's Approve action can push it to the rsync server's authorized_keys. Safe to call at any time (re-run upgrade.ps1, key rotation, etc.). """ auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 body = request.get_json(silent=True) or {} public_key = (body.get("ssh_public_key") or "").strip() if not public_key: return jsonify(error="ssh_public_key required."), 400 conn = _vision_db() try: row = conn.execute( "SELECT machine_id, status FROM vision_agents WHERE agent_token = ? AND revoked = 0", (token,), ).fetchone() if not row: return jsonify(error="Invalid or revoked agent_token."), 401 conn.execute( "UPDATE vision_agents SET ssh_public_key = ? WHERE machine_id = ?", (public_key, row["machine_id"]), ) conn.commit() finally: conn.close() # For already-active agents, push the updated key to the rsync server immediately. # (Pending agents get their key pushed at approval time.) key_pushed = False key_push_error = None if row["status"] == "active": result = _vision_write_ssh_key_to_rsync(public_key) if result is True: key_pushed = True conn2 = _vision_db() try: conn2.execute( "UPDATE vision_agents SET ssh_key_written = 1 WHERE machine_id = ?", (row["machine_id"],), ) conn2.commit() finally: conn2.close() else: key_push_error = result return jsonify(ok=True, key_pushed=key_pushed, key_push_error=key_push_error) @app.route("/vision/checkin", methods=["POST"]) def vision_checkin(): """Agent -> server check-in. Validates bearer agent_token, stores all 14 fields.""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 conn = _vision_db() try: agent = conn.execute( "SELECT machine_id, status, revoked FROM vision_agents WHERE agent_token = ?", (token,), ).fetchone() if not agent: return jsonify(error="Invalid agent_token."), 401 machine_id = agent["machine_id"] agent_status = agent["status"] # Revoked agents: deliver any pending uninstall command so the agent can # self-uninstall cleanly, then signal the revoked status back. if agent["revoked"]: cmd_row = conn.execute( "SELECT id, action, params_json FROM vision_commands" " WHERE machine_id = ? AND acked_at IS NULL" " ORDER BY created_at ASC LIMIT 1", (machine_id,), ).fetchone() if cmd_row: conn.execute( "UPDATE vision_commands SET delivered_at = COALESCE(delivered_at, ?)" " WHERE id = ?", (time.time(), cmd_row["id"]), ) conn.commit() params = None if cmd_row["params_json"]: try: params = json.loads(cmd_row["params_json"]) except (json.JSONDecodeError, TypeError): params = None command = {"id": cmd_row["id"], "action": cmd_row["action"], "params": params} else: command = None return jsonify(ack=True, agent_status="revoked", command=command) body = request.get_json(silent=True) or {} now = time.time() conn.execute( """INSERT INTO vision_checkins (machine_id, received_at, hostname, logged_in_user, os_version, agent_version, status, capture_enabled, last_capture_ts, foreground_app, idle_seconds, cpu_pct, screen_count, agent_uptime_s, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", ( machine_id, now, body.get("hostname"), body.get("logged_in_user"), body.get("os_version"), body.get("agent_version"), body.get("status"), body.get("capture_enabled"), body.get("last_capture_ts"), body.get("foreground_app"), body.get("idle_seconds"), body.get("cpu_pct"), body.get("screen_count"), body.get("agent_uptime_s"), body.get("ts"), ), ) conn.execute( "UPDATE vision_agents SET last_seen_at = ?, hostname = COALESCE(?, hostname)" " WHERE machine_id = ?", (now, body.get("hostname"), machine_id), ) # Dequeue oldest pending command for this machine (acked_at IS NULL). cmd_row = conn.execute( "SELECT id, action, params_json FROM vision_commands" " WHERE machine_id = ? AND acked_at IS NULL" " ORDER BY created_at ASC LIMIT 1", (machine_id,), ).fetchone() if cmd_row: conn.execute( "UPDATE vision_commands SET delivered_at = COALESCE(delivered_at, ?)" " WHERE id = ?", (now, cmd_row["id"]), ) _vision_prune_checkins(conn) conn.commit() finally: conn.close() # First real check-in auto-disables mock mode. settings = _load_vision_settings() if settings.get("mock_data"): settings["mock_data"] = False _save_vision_settings(settings) # Retention sweep over the footage store (throttled to ~1/hour internally). _vision_purge_footage(now) command = None if cmd_row: params = None if cmd_row["params_json"]: try: params = json.loads(cmd_row["params_json"]) except (json.JSONDecodeError, TypeError): params = None command = {"id": cmd_row["id"], "action": cmd_row["action"], "params": params} # Push the machine's effective capture policy (DVI-427) so the agent applies # capture rate / idle threshold / exclusion list / schedule without a new channel. # Also return agent_status so the agent can detect pending→active transitions # without a separate status endpoint (DVI-438). config_delta = _vision_effective_policy(machine_id) return jsonify(ack=True, next_interval_s=60, command=command, config_delta=config_delta, agent_status=agent_status) _VISION_COMMAND_ACTIONS = frozenset({ "pause_capture", "resume_capture", "set_interval", "collect_screenshot", "restart_agent", "upload_log", "start_capture_task", "update_agent", "cleanup_legacy", "uninstall", }) # Fallback used only when the deployed ZIP is missing/unreadable. The live value # is read dynamically from vision_agent.py inside togen-vision-agent.zip so a new # ZIP deploy advertises its version without an app.py edit (DVI-723). _LATEST_VISION_AGENT_VERSION_FALLBACK = "3.2.15" _AGENT_VERSION_RE = re.compile(r'^_AGENT_VERSION\s*=\s*["\']([^"\']+)["\']', re.MULTILINE) # (version, source-zip-mtime) cache so we re-read only when the ZIP changes. _latest_vision_agent_version_cache = {"version": None, "mtime": None} def _latest_vision_agent_version(): """Read _AGENT_VERSION from vision_agent.py inside the deployed agent ZIP. Cached against the ZIP's mtime so a redeploy is picked up without a restart. Falls back to _LATEST_VISION_AGENT_VERSION_FALLBACK if the ZIP is absent or the version line cannot be parsed. """ path = VISION_AGENT_DIST_DIR / VISION_AGENT_ZIP_NAME try: mtime = path.stat().st_mtime except OSError: return _LATEST_VISION_AGENT_VERSION_FALLBACK cache = _latest_vision_agent_version_cache if cache["version"] is not None and cache["mtime"] == mtime: return cache["version"] version = _LATEST_VISION_AGENT_VERSION_FALLBACK try: with zipfile.ZipFile(str(path)) as zf: source = zf.read("vision_agent.py").decode("utf-8", "replace") match = _AGENT_VERSION_RE.search(source) if match: version = match.group(1) except (zipfile.BadZipFile, KeyError, OSError): pass cache["version"] = version cache["mtime"] = mtime return version @app.route("/vision/commands", methods=["POST"]) @togen_admin_required def vision_commands_create(): """Admin enqueues a remote command for a collector machine.""" body = request.get_json(silent=True) or {} machine_id = (body.get("machine_id") or "").strip() action = (body.get("action") or "").strip() params = body.get("params") if not machine_id: return jsonify(error="machine_id required."), 400 if action not in _VISION_COMMAND_ACTIONS: return jsonify(error=f"action must be one of: {', '.join(sorted(_VISION_COMMAND_ACTIONS))}."), 400 cmd_id = "cmd-" + uuid.uuid4().hex[:8] now = time.time() params_json = json.dumps(params) if params is not None else None conn = _vision_db() try: agent = conn.execute( "SELECT machine_id FROM vision_agents WHERE machine_id = ? AND revoked = 0", (machine_id,), ).fetchone() if not agent: return jsonify(error="machine_id not found or revoked."), 404 conn.execute( "INSERT INTO vision_commands (id, machine_id, action, params_json, created_at)" " VALUES (?, ?, ?, ?, ?)", (cmd_id, machine_id, action, params_json, now), ) conn.commit() finally: conn.close() return jsonify(cmd_id=cmd_id), 201 # Max stored log size per (machine, file). Logs are tails for diagnostics, not archives. _VISION_LOG_MAX_CHARS = 200_000 @app.route("/vision/agent-log", methods=["POST"]) def vision_agent_log_upload(): """Agent uploads a log file's content (response to an upload_log command). Authenticated by bearer agent_token. Body: {file, content}. The latest upload per (machine, file) is retained for the admin log viewer (DVI-705 #2).""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 body = request.get_json(silent=True) or {} file = (body.get("file") or "").strip() content = body.get("content") if not file: return jsonify(error="file is required."), 400 if not isinstance(content, str): return jsonify(error="content must be a string."), 400 if len(content) > _VISION_LOG_MAX_CHARS: content = content[-_VISION_LOG_MAX_CHARS:] conn = _vision_db() try: agent = conn.execute( "SELECT machine_id FROM vision_agents WHERE agent_token = ? AND revoked = 0", (token,), ).fetchone() if not agent: return jsonify(error="Invalid or revoked agent_token."), 401 conn.execute( "INSERT INTO vision_agent_logs (machine_id, file, content, received_at)" " VALUES (?, ?, ?, ?)" " ON CONFLICT(machine_id, file) DO UPDATE SET content = excluded.content," " received_at = excluded.received_at", (agent["machine_id"], file, content, time.time()), ) conn.commit() finally: conn.close() return jsonify(ok=True), 201 @app.route("/vision/agent-log") @togen_admin_required def vision_agent_log_get(): """Admin log viewer (DVI-705 #2). With ?machine_id only, lists files that have uploaded content. With ?machine_id&file, returns that file's content + timestamp.""" machine_id = (request.args.get("machine_id") or "").strip() file = (request.args.get("file") or "").strip() if not machine_id: return jsonify(error="machine_id required."), 400 conn = _vision_db() try: if file: row = conn.execute( "SELECT content, received_at FROM vision_agent_logs" " WHERE machine_id = ? AND file = ?", (machine_id, file), ).fetchone() if not row: return jsonify(machine_id=machine_id, file=file, content=None, received_at=None) return jsonify(machine_id=machine_id, file=file, content=row["content"], received_at=row["received_at"]) rows = conn.execute( "SELECT file, received_at FROM vision_agent_logs WHERE machine_id = ?" " ORDER BY file ASC", (machine_id,), ).fetchall() finally: conn.close() return jsonify(machine_id=machine_id, files=[{"file": r["file"], "received_at": r["received_at"]} for r in rows]) @app.route("/vision/agent-logs") @togen_admin_required def vision_agent_logs_list(): """DVI-742: list every uploaded/requested log across all agents for the Vision Logs tab. Joins vision_agent_logs with vision_agents for the hostname and overlays the friendly-name overrides. Newest received first.""" names = _load_vision_agent_names() conn = _vision_db() try: rows = conn.execute( "SELECT l.machine_id, l.file, l.received_at, a.hostname" " FROM vision_agent_logs l" " LEFT JOIN vision_agents a ON a.machine_id = l.machine_id" " ORDER BY l.received_at DESC" ).fetchall() finally: conn.close() logs = [] for r in rows: mid = r["machine_id"] logs.append({ "machine_id": mid, "hostname": r["hostname"] or None, "display_name": names.get(mid) or None, "file": r["file"], "received_at": r["received_at"], }) return jsonify(logs=logs) @app.route("/vision/assistant-config") @vision_access_required def vision_assistant_config_get(): """Return the assistant models available to the Vision user, split into System Providers (org-configured in Togen Admin → Assistant) and User Models (the user's own connection in User Settings → Assistant), plus the user's saved Vision selection. Option values are namespaced "source:modelId".""" from assistant import load_assistant_settings, get_user_assistant, ASSISTANT_PROVIDERS email = session["user"]["email"] user_cfg = _load_user_settings(email) global_cfg = load_assistant_settings() ua = get_user_assistant(user_cfg) system_providers = [] if global_cfg.get("api_key"): for prov in ASSISTANT_PROVIDERS.values(): for m in prov.get("models", []): system_providers.append({ "value": "system:" + m["id"], "label": prov["label"] + " — " + m["label"], }) user_models = [] if ua.get("api_key") and ua.get("model"): model_id = ua["model"] label = model_id provider_label = "" for prov in ASSISTANT_PROVIDERS.values(): for m in prov.get("models", []): if m["id"] == model_id: label = m["label"] provider_label = prov["label"] break user_models.append({ "value": "user:" + model_id, "label": (provider_label + " — " + label) if provider_label else label, }) selected = (user_cfg.get("vision") or {}).get("assistant_model", "") valid = {o["value"] for o in system_providers} | {o["value"] for o in user_models} if selected not in valid: selected = "" return jsonify( system_providers=system_providers, user_models=user_models, selected=selected, ) @app.route("/vision/assistant-config", methods=["POST"]) @vision_access_required def vision_assistant_config_post(): """Persist the Vision user's assistant model selection. Validates the value against the models currently available to the user.""" from assistant import load_assistant_settings, get_user_assistant, ASSISTANT_PROVIDERS email = session["user"]["email"] body = request.get_json(silent=True) or {} model = str(body.get("model", "")).strip() user_cfg = _load_user_settings(email) global_cfg = load_assistant_settings() ua = get_user_assistant(user_cfg) valid = set() if global_cfg.get("api_key"): for prov in ASSISTANT_PROVIDERS.values(): for m in prov.get("models", []): valid.add("system:" + m["id"]) if ua.get("api_key") and ua.get("model"): valid.add("user:" + ua["model"]) if model and model not in valid: return jsonify(error="Selected model is not available."), 400 vision_cfg = dict(user_cfg.get("vision") or {}) vision_cfg["assistant_model"] = model user_cfg["vision"] = vision_cfg _save_user_settings(email, user_cfg) return jsonify(ok=True) @app.route("/vision/commands//ack", methods=["POST"]) def vision_command_ack(cmd_id): """Agent acknowledges execution of a delivered command.""" auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" if not token: return jsonify(error="Missing bearer agent_token."), 401 body = request.get_json(silent=True) or {} status = (body.get("status") or "").strip() if status not in ("ok", "error"): return jsonify(error="status must be 'ok' or 'error'."), 400 error_message = body.get("error_message") or None now = time.time() conn = _vision_db() try: # Allow revoked agents to ack their final commands (e.g. uninstall) so the # command history shows "acked" rather than stuck in "delivered". agent = conn.execute( "SELECT machine_id FROM vision_agents WHERE agent_token = ?", (token,), ).fetchone() if not agent: return jsonify(error="Invalid agent_token."), 401 cmd = conn.execute( "SELECT id FROM vision_commands WHERE id = ? AND machine_id = ?", (cmd_id, agent["machine_id"]), ).fetchone() if not cmd: return jsonify(error="Command not found."), 404 conn.execute( "UPDATE vision_commands SET acked_at = ?, acked_status = ?, error_message = ?" " WHERE id = ?", (now, status, error_message, cmd_id), ) conn.commit() finally: conn.close() return jsonify(ack=True) def _vision_compute_sparkline(checkin_times, now): """Bucket recent check-in timestamps into online(1)/offline(0) buckets over the sparkline window. A bucket is online if any check-in fell within it.""" n = _VISION_SPARK_BUCKETS window = _VISION_SPARK_WINDOW_S start = now - window bucket_s = window / n spark = [0] * n for t in checkin_times: if t < start or t > now: continue idx = int((t - start) / bucket_s) if idx >= n: idx = n - 1 spark[idx] = 1 return spark @app.route("/vision/collectors") @vision_access_required def vision_collectors(): """Return collector machine list. Online/offline derived from last_seen_at vs staleness. Phase-3 (DVI-427) adds per-machine sparkline, uptime/restart indicator, and the latest captured thumbnail/full-frame URLs.""" settings = _load_vision_settings() if settings.get("mock_data"): return jsonify(collectors=_vision_mock_collectors(), mock_data=True, latest_agent_version=_latest_vision_agent_version()) now = time.time() spark_start = now - _VISION_SPARK_WINDOW_S conn = _vision_db() try: rows = conn.execute( """SELECT va.machine_id, va.hostname, va.last_seen_at, vc.logged_in_user, vc.agent_version, vc.status, vc.capture_enabled, vc.foreground_app, vc.idle_seconds, vc.cpu_pct, vc.agent_uptime_s FROM vision_agents va LEFT JOIN ( SELECT machine_id, logged_in_user, agent_version, status, capture_enabled, foreground_app, idle_seconds, cpu_pct, agent_uptime_s FROM vision_checkins WHERE id IN ( SELECT MAX(id) FROM vision_checkins GROUP BY machine_id ) ) vc ON vc.machine_id = va.machine_id WHERE va.status = 'active' ORDER BY va.last_seen_at DESC""" ).fetchall() # History within the sparkline window for trend + restart detection. hist = conn.execute( "SELECT machine_id, received_at, agent_uptime_s FROM vision_checkins" " WHERE received_at >= ? ORDER BY machine_id, received_at ASC", (spark_start,), ).fetchall() finally: conn.close() times_by_machine = {} uptimes_by_machine = {} for h in hist: times_by_machine.setdefault(h["machine_id"], []).append(h["received_at"]) uptimes_by_machine.setdefault(h["machine_id"], []).append(h["agent_uptime_s"]) names = _load_vision_agent_names() policy = _load_vision_policy() out = [] for r in rows: mid = r["machine_id"] last_seen = r["last_seen_at"] cap = r["capture_enabled"] # Restart count = times agent_uptime_s dropped vs the prior check-in. ups = [u for u in uptimes_by_machine.get(mid, []) if u is not None] restarts = sum(1 for a, b in zip(ups, ups[1:]) if b < a) latest = _vision_latest_frame(mid, mock=False) # Capture mode/fps from the machine's effective policy so tiles can show a # video-vs-screenshot badge; latest_kind is the actual newest captured type. eff_pol = _vision_effective_policy(mid, policy) out.append({ "machine_id": mid, "hostname": r["hostname"], "display_name": names.get(mid) or None, "logged_in_user": r["logged_in_user"], "agent_version": r["agent_version"], "last_seen": int(last_seen) if last_seen else None, "online": bool(last_seen) and (now - last_seen) <= VISION_STALE_SECONDS, "status": r["status"], "capture_enabled": None if cap is None else bool(cap), "foreground_app": r["foreground_app"], "idle_seconds": r["idle_seconds"], "cpu_pct": r["cpu_pct"], "agent_uptime_s": r["agent_uptime_s"], "restarts": restarts, "sparkline": _vision_compute_sparkline(times_by_machine.get(mid, []), now), "thumb_url": latest["thumb_url"] if latest else None, "full_url": latest["full_url"] if latest else None, "capture_mode": eff_pol.get("capture_mode"), "capture_fps": eff_pol.get("capture_fps"), "latest_kind": latest.get("kind") if latest else None, }) return jsonify(collectors=out, mock_data=False, latest_agent_version=_latest_vision_agent_version()) @app.route("/vision/agent-name", methods=["POST"]) @togen_admin_required def vision_agent_name_post(): """Set or clear a friendly display-name override for a machine (DVI-682). Body: {machine_id, name}. An empty name clears the override (reverts to hostname).""" body = request.get_json(silent=True) or {} mid = (body.get("machine_id") or "").strip() name = (body.get("name") or "").strip() if not mid: return jsonify(error="machine_id is required."), 400 if len(name) > 80: return jsonify(error="Name must be 80 characters or fewer."), 400 with _vision_names_lock: names = _load_vision_agent_names() if name: names[mid] = name else: names.pop(mid, None) _save_vision_agent_names(names) return jsonify(ok=True, machine_id=mid, name=name or None) @app.route("/togen-admin/vision-settings") @togen_admin_required def vision_settings_get(): return jsonify(settings=_load_vision_settings()) @app.route("/togen-admin/vision-settings", methods=["POST"]) @togen_admin_required def vision_settings_post(): body = request.get_json(silent=True) or {} settings = _load_vision_settings() if "mock_data" in body: settings["mock_data"] = bool(body["mock_data"]) _save_vision_settings(settings) return jsonify(ok=True, settings=settings) @app.route("/togen-admin/vision-policy") @togen_admin_required def vision_policy_get(): """Return the capture policy (default + per-machine overrides) plus the field defaults so the UI can render an empty/reset form.""" return jsonify(policy=_load_vision_policy(), defaults=_VISION_POLICY_DEFAULTS) @app.route("/togen-admin/vision-policy", methods=["POST"]) @togen_admin_required def vision_policy_post(): """Persist the capture policy. Body: {default?: {...}, machines?: {id: {...}}}. A machine override of null/{} removes that override.""" body = request.get_json(silent=True) or {} policy = _load_vision_policy() if isinstance(body.get("default"), dict): merged = dict(policy["default"]) merged.update(_vision_clean_policy(body["default"])) policy["default"] = merged if isinstance(body.get("machines"), dict): for mid, ov in body["machines"].items(): mid = str(mid).strip() if not mid: continue if ov in (None, {}): policy["machines"].pop(mid, None) elif isinstance(ov, dict): policy["machines"][mid] = _vision_clean_policy(ov) _save_vision_policy(policy) return jsonify(ok=True, policy=policy) @app.route("/vision/frames//") @vision_access_required def vision_frame(machine_id, frame_id): """Serve a captured frame (D2). ?size=thumb|full. Mock frames (mock mode or mock- ids) are rendered as self-describing SVG placeholders; real frames are read from the configurable footage store, thumbnail companion preferred.""" machine_id = (machine_id or "").strip() frame_id = (frame_id or "").strip() size = (request.args.get("size") or "full").strip().lower() if size not in ("thumb", "full"): size = "full" if not _VISION_FRAME_ID_RE.match(machine_id) or not _VISION_FRAME_ID_RE.match(frame_id): return jsonify(error="Invalid identifier."), 400 if _load_vision_settings().get("mock_data") or frame_id.startswith("mock-"): svg = _vision_mock_frame_svg(machine_id, frame_id, size) return svg, 200, {"Content-Type": "image/svg+xml", "Cache-Control": "no-store"} # Quarantine: never serve footage for a machine that isn't approved/active. if not _vision_machine_active(machine_id): return jsonify(error="Frame not found."), 404 resp = _footage_store.frame_bytes(machine_id, frame_id, size) if resp is None: return jsonify(error="Frame not found."), 404 return resp @app.route("/vision/segments//") @vision_access_required def vision_segment(machine_id, frame_id): """Serve a captured H.264 video segment (DVI-727). Same identifier/quarantine guards as frame serving; send_file honors HTTP Range so the Phase-3 HTML5