#!/opt/imunify360/venv/bin/python3
"""This module import peewee_migrate and apply migrations, for Imunify-AV
it's entrypoint for service"""
import contextlib
import os
import sys
import signal
import subprocess
import threading
import time
from collections.abc import Iterable
from logging import getLogger
from sqlite3 import Error as SqliteError, connect
from peewee_migrate import migrator
from playhouse.sqlite_ext import SqliteExtDatabase
import defence360agent.internals.logger
from defence360agent.application import app
from defence360agent.application.settings import configure
from defence360agent.contracts.config import Core
from defence360agent.contracts.config import Model
from defence360agent.router import Router
from defence360agent.subsys.svcctl import AGENT_SERVICE_NAME
from defence360agent.subsys import systemd_notifier
from defence360agent.model.instance import db as db_instance
from defence360agent.model import tls_check
from defence360agent.utils import (
write_pid_file,
IM360_RESIDENT_PID_PATH,
cleanup_pid_file,
)
from defence360agent.utils.check_db import (
make_backup,
mark_with_timestamp,
recreate_schema_models,
)
logger = getLogger(__name__)
GO_SERVICE_NAME = "/usr/bin/imunify-resident"
# Shipped with the interpreter this agent runs on; .recover needs >= 3.29.
SQLITE_CLI = "/opt/alt/sqlite/usr/bin/sqlite3"
# Measured at ~5 MB/s for recover plus reload, so this covers a database far
# larger than any seen; three of them still fit the unit's start timeout.
SALVAGE_TIMEOUT_SECS = 180
SYSTEMCTL_TIMEOUT_SECS = 30
# The primary sqlite result codes that prove the file itself is unreadable.
SQLITE_ERROR = 1 # a header this library cannot parse: unsupported file format
SQLITE_CORRUPT = 11 # database disk image is malformed
SQLITE_NOTADB = 26 # file is not a database
# Any other failure - a lock, an I/O error, a full disk - means the check could
# not be made rather than that the file is damaged, and acting on that verdict
# would delete a healthy database, so leaving it alone is the safe default.
# Compared on the primary code because Python surfaces extended ones:
# SQLITE_BUSY_SNAPSHOT arrives as 517 and matches no primary name.
DAMAGED_SQLITE_CODES = frozenset((SQLITE_ERROR, SQLITE_CORRUPT, SQLITE_NOTADB))
@contextlib.contextmanager
def exc_handler(log_msg: str, reraise: bool):
"""
Logs error in case of exception.
Depending on `reraise`:
- re-raise exception and don't include exception info in the log operation
- do not re-raise exception and include exception info in the log operation
"""
try:
yield
except Exception:
logger.error(log_msg, exc_info=not reraise)
if reraise:
raise
def apply_migrations(db: SqliteExtDatabase, migrations_dirs: Iterable[str]):
"""Apply migrations: restructure db, config files, etc."""
router = Router(
db,
migrations_dirs=migrations_dirs,
logger=logger,
)
# HACK: Migrator uses global unconfigurable LOGGER,
# overrride it, to use our logging settings
migrator.LOGGER = logger
router.run()
def _park_sidecars(db_path: str, backup: str) -> None:
# -wal and -shm belong to the file that was just replaced, and leaving them
# behind makes SQLite reopen the old content through them. They can also
# hold committed rows that are in no other file, so they move next to the
# backup rather than being deleted.
for suffix in ("-wal", "-shm"):
with contextlib.suppress(OSError):
os.replace(db_path + suffix, backup + suffix)
def _is_corrupted(db_path: str) -> bool:
"""True only on positive evidence that the file itself is damaged."""
# Deliberately stricter than check_db.is_db_corrupted, which reports any
# DatabaseError as corruption. That is fine for the interactive checkdb
# path; here the verdict deletes a file, so "could not check" must not
# become "damaged".
try:
with contextlib.closing(connect(db_path)) as conn:
# Opening and closing a database in WAL mode checkpoints its -wal
# into the file, so this check is not read-only.
verdict = conn.execute("PRAGMA integrity_check").fetchone()
return verdict is not None and "ok" not in verdict
except SqliteError as e:
if getattr(e, "sqlite_errorcode", 0) & 0xFF in DAMAGED_SQLITE_CODES:
# A destructive verdict has to say what drove it: these codes
# cannot tell a damaged file from one written in a format this
# library cannot read, so the code belongs in the record.
logger.warning(
"Database %s reported %s (%s); treating it as damaged",
db_path,
getattr(e, "sqlite_errorname", "an unnamed sqlite error"),
e,
)
return True
logger.warning(
"Cannot check %s for corruption (%s); leaving it alone",
db_path,
e,
)
return False
def _count_rows(db_path: str) -> int:
# Counted on the rebuilt file rather than on the .recover output: a load
# that only partly succeeded must not be reported as fully recovered.
# lost_and_found holds rows .recover could not attribute to a table, which
# is what it produces when the schema page itself is unreadable. Nothing
# reads them, so they must not be counted as recovered either.
with contextlib.closing(connect(db_path)) as conn:
tables = conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
" AND name NOT LIKE 'lost_and_found%'"
" AND name NOT LIKE 'sqlite_%'"
).fetchall()
return sum(
conn.execute(
'SELECT count(*) FROM "%s"' % name.replace('"', '""')
).fetchone()[0]
for (name,) in tables
)
def _salvage_database(db_path: str) -> int | None:
"""Rebuild db_path from readable pages; rows kept, else None."""
# iterdump() aborts on the first damaged page. The CLI's .recover walks the
# b-tree directly and skips what it cannot read, so it is the only salvage
# that gets anything off a malformed file.
if not os.path.exists(SQLITE_CLI):
return None
recovered_sql = mark_with_timestamp(db_path, extension="recover.sql")
rebuilt = db_path + ".rebuilt"
# A rebuilt file left by a previous attempt that was killed mid-flight
# would be loaded into, mixing stale content into the result.
with contextlib.suppress(OSError):
os.remove(rebuilt)
try:
with open(recovered_sql, "wb") as sql:
subprocess.run(
[SQLITE_CLI, db_path, ".recover"],
stdout=sql,
stderr=subprocess.DEVNULL,
timeout=SALVAGE_TIMEOUT_SECS,
check=True,
)
with open(recovered_sql, "rb") as sql:
subprocess.run(
[SQLITE_CLI, rebuilt],
stdin=sql,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=SALVAGE_TIMEOUT_SECS,
check=True,
)
if _is_corrupted(rebuilt):
return None
rows = _count_rows(rebuilt)
if rows == 0:
return None
os.replace(rebuilt, db_path)
return rows
except (OSError, subprocess.SubprocessError, SqliteError):
return None
finally:
for path in (rebuilt, recovered_sql):
with contextlib.suppress(OSError):
os.remove(path)
def _restore_wal(db_path: str) -> None:
# A repaired file is a new one, and new SQLite files default to the DELETE
# journal. Both agents share these databases and only WAL keeps a reader
# from being blocked by a writer. The resident re-applies WAL to its main
# database on connect but not to the ones it attaches, so a repaired
# attached file would stay in DELETE for the life of the installation.
try:
with contextlib.closing(connect(db_path)) as conn:
conn.execute("PRAGMA journal_mode=WAL")
except SqliteError as e:
logger.error("Could not restore WAL mode on %s: %s", db_path, e)
def _restart_non_resident_agent() -> None:
# A repair replaces the file's inode. The non-resident agent is a separate
# long-lived process that attaches the same databases, and it is not
# restarted with the resident: the units share only an ordering, so
# nothing propagates. Left alone it keeps writing into the file it still
# holds open, which no longer has a name, and those rows are lost while
# the resident sees a database without them. try-restart leaves a stopped
# unit stopped, and --no-block does not wait on a job that is ordered
# after the startup issuing it.
try:
subprocess.run(
[
"systemctl",
"try-restart",
"--no-block",
"%s.service" % AGENT_SERVICE_NAME,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=SYSTEMCTL_TIMEOUT_SECS,
check=True,
)
except (OSError, subprocess.SubprocessError) as e:
logger.warning(
"Could not restart %s after repairing a database: %s",
AGENT_SERVICE_NAME,
e,
)
def _repair_corrupted_attached_dbs(
attached_dbs: tuple[tuple[str, str], ...],
) -> None:
"""Salvage damaged attached databases, else drop them for ATTACH."""
# The recreation below cannot rewrite a damaged page, so without this the
# file stays broken for the life of the installation.
repaired = False
for db_path, _ in attached_dbs:
if not os.path.exists(db_path) or not _is_corrupted(db_path):
continue
backup = make_backup(db_path)
if backup is None:
logger.error(
"Database %s is corrupted and its backup could not be"
" written; leaving it untouched",
db_path,
)
continue
rows = _salvage_database(db_path)
if rows is None:
try:
os.remove(db_path)
except OSError as e:
logger.error(
"Database %s is corrupted and could not be removed for"
" recreation (%s); leaving it as it is. Backup: %s",
db_path,
e,
backup,
)
continue
_park_sidecars(db_path, backup)
_restore_wal(db_path)
repaired = True
if rows is not None:
# Rows on damaged pages are gone; only the backup still has them.
logger.warning(
"Database %s was corrupted, rebuilt with %d recovered rows."
" Backup: %s",
db_path,
rows,
backup,
)
elif not os.path.exists(SQLITE_CLI):
logger.error(
"Database %s is corrupted and %s is not present to salvage"
" it; recreating it empty. Backup: %s",
db_path,
SQLITE_CLI,
backup,
)
else:
logger.error(
"Database %s is corrupted and could not be salvaged;"
" recreating it empty. Backup: %s",
db_path,
backup,
)
if repaired:
_restart_non_resident_agent()
def prepare_databases(
migrations_dirs: Iterable[str],
attached_dbs: tuple[tuple[str, str], ...] = tuple(),
):
"""
Apply migrations and recreate attached databases.
The workflow:
1. Apply migrations
2. Regardless whether the migrations were applied - recreate attached databases
3. If the recreation of the attached databases was successful - apply migrations again
- this is done to verify that migrations will successfully apply in future for the recreated databases
- the recreation + the migrations in this step are within the same transaction,
so databases will only be recreated if the migrations can applied after the recreation.
"""
# prepare database to operate in WAL journal_mode and run migrations
tls_check.reset()
_repair_corrupted_attached_dbs(attached_dbs)
db_instance.init(Model.PATH)
attached_schemas = []
for db_path, schema_name in attached_dbs:
db_instance.execute_sql("ATTACH ? AS ?", (db_path, schema_name))
attached_schemas.append(schema_name)
try:
logger.info("Applying database migrations...")
systemd_notifier.notify(systemd_notifier.AgentState.MIGRATING)
with db_instance.atomic("EXCLUSIVE"), exc_handler(
"Error applying migrations", reraise=False
):
apply_migrations(db_instance, migrations_dirs)
logger.info("Recreating attached databases...")
with db_instance.atomic("EXCLUSIVE"), exc_handler(
"Error recreating attached databases", reraise=True
):
# Migration history is stored in main db, so to automatically recreate
# attached dbs it is required to recreate schema for them from models
recreate_schema_models(db_instance, attached_schemas)
# verify migrations can be applied after the attached dbs recreation
with exc_handler(
"Error applying migrations after recreating attached"
" databases",
reraise=True,
):
apply_migrations(db_instance, migrations_dirs)
finally:
# close connection immediately since later this process
# will be replaced by execv
db_instance.close()
# required in case package manager or user sends signals while migrations are still running
def signal_handler(sig, _):
logger.warning("Received signal %s in signal_handler", sig)
logger.warning(
"waiting %d seconds so that migrations can finish",
Core.SIGNAL_HANDLER_MIGRATION_TIMEOUT_SECS,
)
time.sleep(Core.SIGNAL_HANDLER_MIGRATION_TIMEOUT_SECS)
logger.info("Exiting")
sys.exit(0)
def run(*, start_pkg="defence360agent", configure=configure):
"""Entry point for Imunify-AV service. Apply migrations,
and then replace process with {start_pkg}.run module."""
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
signal.signal(sig, signal_handler)
try:
if start_pkg == "im360.run_resident":
write_pid_file(IM360_RESIDENT_PID_PATH)
os.umask(Core.FILE_UMASK)
configure()
defence360agent.internals.logger.reconfigure()
migration_thread = threading.Thread(
target=prepare_databases,
args=(app.MIGRATIONS_DIRS, app.MIGRATIONS_ATTACHED_DBS),
)
migration_thread.start()
migration_thread.join()
systemd_notifier.notify(systemd_notifier.AgentState.READY)
logger.info("Starting main process...")
systemd_notifier.notify(systemd_notifier.AgentState.STARTING)
if start_pkg == "im360.run_resident":
Core.GO_FLAG_FILE.touch(exist_ok=True)
logger.info("Run imunify-resident service")
os.execv(
GO_SERVICE_NAME,
[
GO_SERVICE_NAME,
]
+ sys.argv[1:],
)
else:
os.execv(
sys.executable,
[sys.executable, "-m", "{}".format(start_pkg)] + sys.argv[1:],
)
except Exception:
if start_pkg == "im360.run_resident":
cleanup_pid_file(IM360_RESIDENT_PID_PATH)
if __name__ == "__main__":
run()
|