fixes and refactor
This commit is contained in:
HotSwapp
@@ -2,8 +2,7 @@
|
||||
Database configuration and session management
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker, Session
|
||||
from typing import Generator
|
||||
|
||||
from app.config import settings
|
||||
|
||||
248
app/database/fts.py
Normal file
248
app/database/fts.py
Normal file
@@ -0,0 +1,248 @@
|
||||
"""
|
||||
SQLite Full-Text Search (FTS5) helpers.
|
||||
|
||||
Creates and maintains FTS virtual tables and triggers to keep them in sync
|
||||
with their content tables. Designed to be called at app startup.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def _execute_ignore_errors(engine: Engine, sql: str) -> None:
|
||||
"""Execute SQL, ignoring operational errors (e.g., when FTS5 is unavailable)."""
|
||||
from sqlalchemy.exc import OperationalError
|
||||
with engine.begin() as conn:
|
||||
try:
|
||||
conn.execute(text(sql))
|
||||
except OperationalError:
|
||||
# Likely FTS5 extension not available in this SQLite build
|
||||
pass
|
||||
|
||||
|
||||
def ensure_rolodex_fts(engine: Engine) -> None:
|
||||
"""Ensure the `rolodex_fts` virtual table and triggers exist and are populated.
|
||||
|
||||
This uses content=rolodex so the FTS table shadows the base table and is kept
|
||||
in sync via triggers.
|
||||
"""
|
||||
# Create virtual table (if FTS5 is available)
|
||||
_create_table = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS rolodex_fts USING fts5(
|
||||
id,
|
||||
first,
|
||||
last,
|
||||
city,
|
||||
email,
|
||||
memo,
|
||||
content='rolodex',
|
||||
content_rowid='rowid'
|
||||
);
|
||||
"""
|
||||
_execute_ignore_errors(engine, _create_table)
|
||||
|
||||
# Triggers to keep FTS in sync
|
||||
_triggers = [
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS rolodex_ai AFTER INSERT ON rolodex BEGIN
|
||||
INSERT INTO rolodex_fts(rowid, id, first, last, city, email, memo)
|
||||
VALUES (new.rowid, new.id, new.first, new.last, new.city, new.email, new.memo);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS rolodex_ad AFTER DELETE ON rolodex BEGIN
|
||||
INSERT INTO rolodex_fts(rolodex_fts, rowid, id, first, last, city, email, memo)
|
||||
VALUES ('delete', old.rowid, old.id, old.first, old.last, old.city, old.email, old.memo);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS rolodex_au AFTER UPDATE ON rolodex BEGIN
|
||||
INSERT INTO rolodex_fts(rolodex_fts, rowid, id, first, last, city, email, memo)
|
||||
VALUES ('delete', old.rowid, old.id, old.first, old.last, old.city, old.email, old.memo);
|
||||
INSERT INTO rolodex_fts(rowid, id, first, last, city, email, memo)
|
||||
VALUES (new.rowid, new.id, new.first, new.last, new.city, new.email, new.memo);
|
||||
END;
|
||||
""",
|
||||
]
|
||||
for trig in _triggers:
|
||||
_execute_ignore_errors(engine, trig)
|
||||
|
||||
# Backfill if the FTS table exists but is empty
|
||||
with engine.begin() as conn:
|
||||
try:
|
||||
count_fts = conn.execute(text("SELECT count(*) FROM rolodex_fts")).scalar() # type: ignore
|
||||
if count_fts == 0:
|
||||
# Populate from existing rolodex rows
|
||||
conn.execute(text(
|
||||
"""
|
||||
INSERT INTO rolodex_fts(rowid, id, first, last, city, email, memo)
|
||||
SELECT rowid, id, first, last, city, email, memo FROM rolodex;
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
# If FTS table doesn't exist or any error occurs, ignore silently
|
||||
pass
|
||||
|
||||
|
||||
def ensure_files_fts(engine: Engine) -> None:
|
||||
"""Ensure the `files_fts` virtual table and triggers exist and are populated."""
|
||||
_create_table = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS files_fts USING fts5(
|
||||
file_no,
|
||||
id,
|
||||
regarding,
|
||||
file_type,
|
||||
memo,
|
||||
content='files',
|
||||
content_rowid='rowid'
|
||||
);
|
||||
"""
|
||||
_execute_ignore_errors(engine, _create_table)
|
||||
|
||||
_triggers = [
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS files_ai AFTER INSERT ON files BEGIN
|
||||
INSERT INTO files_fts(rowid, file_no, id, regarding, file_type, memo)
|
||||
VALUES (new.rowid, new.file_no, new.id, new.regarding, new.file_type, new.memo);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS files_ad AFTER DELETE ON files BEGIN
|
||||
INSERT INTO files_fts(files_fts, rowid, file_no, id, regarding, file_type, memo)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.id, old.regarding, old.file_type, old.memo);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS files_au AFTER UPDATE ON files BEGIN
|
||||
INSERT INTO files_fts(files_fts, rowid, file_no, id, regarding, file_type, memo)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.id, old.regarding, old.file_type, old.memo);
|
||||
INSERT INTO files_fts(rowid, file_no, id, regarding, file_type, memo)
|
||||
VALUES (new.rowid, new.file_no, new.id, new.regarding, new.file_type, new.memo);
|
||||
END;
|
||||
""",
|
||||
]
|
||||
for trig in _triggers:
|
||||
_execute_ignore_errors(engine, trig)
|
||||
|
||||
with engine.begin() as conn:
|
||||
try:
|
||||
count_fts = conn.execute(text("SELECT count(*) FROM files_fts")).scalar() # type: ignore
|
||||
if count_fts == 0:
|
||||
conn.execute(text(
|
||||
"""
|
||||
INSERT INTO files_fts(rowid, file_no, id, regarding, file_type, memo)
|
||||
SELECT rowid, file_no, id, regarding, file_type, memo FROM files;
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_ledger_fts(engine: Engine) -> None:
|
||||
"""Ensure the `ledger_fts` virtual table and triggers exist and are populated."""
|
||||
_create_table = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS ledger_fts USING fts5(
|
||||
file_no,
|
||||
t_code,
|
||||
note,
|
||||
empl_num,
|
||||
content='ledger',
|
||||
content_rowid='rowid'
|
||||
);
|
||||
"""
|
||||
_execute_ignore_errors(engine, _create_table)
|
||||
|
||||
_triggers = [
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS ledger_ai AFTER INSERT ON ledger BEGIN
|
||||
INSERT INTO ledger_fts(rowid, file_no, t_code, note, empl_num)
|
||||
VALUES (new.rowid, new.file_no, new.t_code, new.note, new.empl_num);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS ledger_ad AFTER DELETE ON ledger BEGIN
|
||||
INSERT INTO ledger_fts(ledger_fts, rowid, file_no, t_code, note, empl_num)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.t_code, old.note, old.empl_num);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS ledger_au AFTER UPDATE ON ledger BEGIN
|
||||
INSERT INTO ledger_fts(ledger_fts, rowid, file_no, t_code, note, empl_num)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.t_code, old.note, old.empl_num);
|
||||
INSERT INTO ledger_fts(rowid, file_no, t_code, note, empl_num)
|
||||
VALUES (new.rowid, new.file_no, new.t_code, new.note, new.empl_num);
|
||||
END;
|
||||
""",
|
||||
]
|
||||
for trig in _triggers:
|
||||
_execute_ignore_errors(engine, trig)
|
||||
|
||||
with engine.begin() as conn:
|
||||
try:
|
||||
count_fts = conn.execute(text("SELECT count(*) FROM ledger_fts")).scalar() # type: ignore
|
||||
if count_fts == 0:
|
||||
conn.execute(text(
|
||||
"""
|
||||
INSERT INTO ledger_fts(rowid, file_no, t_code, note, empl_num)
|
||||
SELECT rowid, file_no, t_code, note, empl_num FROM ledger;
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_qdros_fts(engine: Engine) -> None:
|
||||
"""Ensure the `qdros_fts` virtual table and triggers exist and are populated."""
|
||||
_create_table = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS qdros_fts USING fts5(
|
||||
file_no,
|
||||
form_name,
|
||||
pet,
|
||||
res,
|
||||
case_number,
|
||||
content='qdros',
|
||||
content_rowid='rowid'
|
||||
);
|
||||
"""
|
||||
_execute_ignore_errors(engine, _create_table)
|
||||
|
||||
_triggers = [
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS qdros_ai AFTER INSERT ON qdros BEGIN
|
||||
INSERT INTO qdros_fts(rowid, file_no, form_name, pet, res, case_number)
|
||||
VALUES (new.rowid, new.file_no, new.form_name, new.pet, new.res, new.case_number);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS qdros_ad AFTER DELETE ON qdros BEGIN
|
||||
INSERT INTO qdros_fts(qdros_fts, rowid, file_no, form_name, pet, res, case_number)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.form_name, old.pet, old.res, old.case_number);
|
||||
END;
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS qdros_au AFTER UPDATE ON qdros BEGIN
|
||||
INSERT INTO qdros_fts(qdros_fts, rowid, file_no, form_name, pet, res, case_number)
|
||||
VALUES ('delete', old.rowid, old.file_no, old.form_name, old.pet, old.res, old.case_number);
|
||||
INSERT INTO qdros_fts(rowid, file_no, form_name, pet, res, case_number)
|
||||
VALUES (new.rowid, new.file_no, new.form_name, new.pet, new.res, new.case_number);
|
||||
END;
|
||||
""",
|
||||
]
|
||||
for trig in _triggers:
|
||||
_execute_ignore_errors(engine, trig)
|
||||
|
||||
with engine.begin() as conn:
|
||||
try:
|
||||
count_fts = conn.execute(text("SELECT count(*) FROM qdros_fts")).scalar() # type: ignore
|
||||
if count_fts == 0:
|
||||
conn.execute(text(
|
||||
"""
|
||||
INSERT INTO qdros_fts(rowid, file_no, form_name, pet, res, case_number)
|
||||
SELECT rowid, file_no, form_name, pet, res, case_number FROM qdros;
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
31
app/database/indexes.py
Normal file
31
app/database/indexes.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Database secondary indexes helper.
|
||||
|
||||
Creates small B-tree indexes for common equality filters to speed up searches.
|
||||
Uses CREATE INDEX IF NOT EXISTS so it is safe to call repeatedly at startup
|
||||
and works for existing databases without running a migration.
|
||||
"""
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def ensure_secondary_indexes(engine: Engine) -> None:
|
||||
statements = [
|
||||
# Files
|
||||
"CREATE INDEX IF NOT EXISTS idx_files_status ON files(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_files_file_type ON files(file_type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_files_empl_num ON files(empl_num)",
|
||||
# Ledger
|
||||
"CREATE INDEX IF NOT EXISTS idx_ledger_t_type ON ledger(t_type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_ledger_empl_num ON ledger(empl_num)",
|
||||
]
|
||||
with engine.begin() as conn:
|
||||
for stmt in statements:
|
||||
try:
|
||||
conn.execute(text(stmt))
|
||||
except Exception:
|
||||
# Ignore failures (e.g., non-SQLite engines that still support IF NOT EXISTS;
|
||||
# if not supported, users should manage indexes via migrations)
|
||||
pass
|
||||
|
||||
|
||||
130
app/database/schema_updates.py
Normal file
130
app/database/schema_updates.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Lightweight, idempotent schema updates for SQLite.
|
||||
|
||||
Adds newly introduced columns to existing tables when running on an
|
||||
already-initialized database. Safe to call multiple times.
|
||||
"""
|
||||
from typing import Dict
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
def _existing_columns(conn, table: str) -> set[str]:
|
||||
rows = conn.execute(f"PRAGMA table_info('{table}')").fetchall()
|
||||
return {row[1] for row in rows} # name is column 2
|
||||
|
||||
|
||||
def ensure_schema_updates(engine: Engine) -> None:
|
||||
"""Ensure missing columns are added for backward-compatible updates."""
|
||||
# Map of table -> {column: SQL type}
|
||||
updates: Dict[str, Dict[str, str]] = {
|
||||
# Forms
|
||||
"form_index": {
|
||||
"keyword": "TEXT",
|
||||
},
|
||||
# Richer Life/Number tables (forms & pensions harmonized)
|
||||
"life_tables": {
|
||||
"le_aa": "FLOAT",
|
||||
"na_aa": "FLOAT",
|
||||
"le_am": "FLOAT",
|
||||
"na_am": "FLOAT",
|
||||
"le_af": "FLOAT",
|
||||
"na_af": "FLOAT",
|
||||
"le_wa": "FLOAT",
|
||||
"na_wa": "FLOAT",
|
||||
"le_wm": "FLOAT",
|
||||
"na_wm": "FLOAT",
|
||||
"le_wf": "FLOAT",
|
||||
"na_wf": "FLOAT",
|
||||
"le_ba": "FLOAT",
|
||||
"na_ba": "FLOAT",
|
||||
"le_bm": "FLOAT",
|
||||
"na_bm": "FLOAT",
|
||||
"le_bf": "FLOAT",
|
||||
"na_bf": "FLOAT",
|
||||
"le_ha": "FLOAT",
|
||||
"na_ha": "FLOAT",
|
||||
"le_hm": "FLOAT",
|
||||
"na_hm": "FLOAT",
|
||||
"le_hf": "FLOAT",
|
||||
"na_hf": "FLOAT",
|
||||
"table_year": "INTEGER",
|
||||
"table_type": "VARCHAR(45)",
|
||||
},
|
||||
"number_tables": {
|
||||
"month": "INTEGER",
|
||||
"na_aa": "FLOAT",
|
||||
"na_am": "FLOAT",
|
||||
"na_af": "FLOAT",
|
||||
"na_wa": "FLOAT",
|
||||
"na_wm": "FLOAT",
|
||||
"na_wf": "FLOAT",
|
||||
"na_ba": "FLOAT",
|
||||
"na_bm": "FLOAT",
|
||||
"na_bf": "FLOAT",
|
||||
"na_ha": "FLOAT",
|
||||
"na_hm": "FLOAT",
|
||||
"na_hf": "FLOAT",
|
||||
"table_type": "VARCHAR(45)",
|
||||
"description": "TEXT",
|
||||
},
|
||||
"form_list": {
|
||||
"status": "VARCHAR(45)",
|
||||
},
|
||||
# Printers: add advanced legacy fields
|
||||
"printers": {
|
||||
"number": "INTEGER",
|
||||
"page_break": "VARCHAR(50)",
|
||||
"setup_st": "VARCHAR(200)",
|
||||
"reset_st": "VARCHAR(200)",
|
||||
"b_underline": "VARCHAR(100)",
|
||||
"e_underline": "VARCHAR(100)",
|
||||
"b_bold": "VARCHAR(100)",
|
||||
"e_bold": "VARCHAR(100)",
|
||||
"phone_book": "BOOLEAN",
|
||||
"rolodex_info": "BOOLEAN",
|
||||
"envelope": "BOOLEAN",
|
||||
"file_cabinet": "BOOLEAN",
|
||||
"accounts": "BOOLEAN",
|
||||
"statements": "BOOLEAN",
|
||||
"calendar": "BOOLEAN",
|
||||
},
|
||||
# Pensions
|
||||
"pension_schedules": {
|
||||
"vests_on": "DATE",
|
||||
"vests_at": "FLOAT",
|
||||
},
|
||||
"marriage_history": {
|
||||
"married_from": "DATE",
|
||||
"married_to": "DATE",
|
||||
"married_years": "FLOAT",
|
||||
"service_from": "DATE",
|
||||
"service_to": "DATE",
|
||||
"service_years": "FLOAT",
|
||||
"marital_percent": "FLOAT",
|
||||
},
|
||||
"death_benefits": {
|
||||
"lump1": "FLOAT",
|
||||
"lump2": "FLOAT",
|
||||
"growth1": "FLOAT",
|
||||
"growth2": "FLOAT",
|
||||
"disc1": "FLOAT",
|
||||
"disc2": "FLOAT",
|
||||
},
|
||||
}
|
||||
|
||||
with engine.begin() as conn:
|
||||
for table, cols in updates.items():
|
||||
try:
|
||||
existing = _existing_columns(conn, table)
|
||||
except Exception:
|
||||
# Table may not exist yet
|
||||
continue
|
||||
for col_name, col_type in cols.items():
|
||||
if col_name not in existing:
|
||||
try:
|
||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col_name} {col_type}")
|
||||
except Exception:
|
||||
# Ignore if not applicable (other engines) or race condition
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user