diff --git a/README.rst b/README.rst index e0ae56f51..75bd62bde 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,7 @@ For more details: --single-connection Do not use a separate connection for completions. -v, --version Version of pgcli. -d, --dbname TEXT database name to connect to. + --env-file PATH Read a database URL from a local .env file. --pgclirc FILE Location of pgclirc file. -D, --dsn TEXT Use DSN configured into the [alias_dsn] section of pgclirc file. @@ -111,6 +112,30 @@ For more details: ``pgcli`` also supports many of the same `environment variables`_ as ``psql`` for login options (e.g. ``PGHOST``, ``PGPORT``, ``PGUSER``, ``PGPASSWORD``, ``PGDATABASE``). +To connect using a URL in a local project file, opt in explicitly:: + + # .env (or .env.local) + DB_URL='postgresql://user:password@localhost:5432/app_db' + + $ pgcli --env-file .env + $ pgcli --env-file .env.local --ping + +The first non-empty variable wins, in this order: ``DATABASE_URL``, +``DATABASE_URI``, ``DB_URL``, ``DB_URI``. Values must be ``postgres://`` or +``postgresql://`` URLs. Quoted values, comments, and ``export`` prefixes are +supported. Variable references such as ``${PASSWORD}`` are kept literal. + +Explicit connection arguments (``DBNAME``, ``USERNAME``, ``-d``, ``-h``, ``-p``, +``-U``/``-u``, or ``-D``) select the usual connection instead of the file URL. +Otherwise, the file URL takes precedence over environment defaults such as +``PGDATABASE`` and ``DSN``. The usual libpq defaults, including ``PGSERVICE``, +still apply to the URL. Other options, including ``--ssh-tunnel``, +``--list``, and ``--ping``, work as with a URL passed on the command line. + +Only the named file is read; pgcli does not search parent directories or load +``.env`` automatically. An unreadable file, missing URL, or invalid URL produces +an error. The file does not export any variables into the process environment. + The SSL-related environment variables are also supported, so if you need to connect a postgres database via ssl connection, you can set set environment like this: :: diff --git a/changelog.rst b/changelog.rst index fc362092b..92674d384 100644 --- a/changelog.rst +++ b/changelog.rst @@ -3,6 +3,9 @@ Upcoming Features: --------- +* Add ``--env-file PATH`` to opt in to reading a database URL from a local + ``.env`` file. Explicit connection arguments take precedence over the file, + which takes precedence over environment defaults. * Add ``\\cd`` and ``\\ls`` filesystem commands, with path completion for filesystem-aware commands including ``\\e``, ``\\i``, ``\\log-file``, and ``\\o``. * Add a ``\service`` prompt token for connections made using ``service=...`` diff --git a/pgcli-completion.bash b/pgcli-completion.bash index 620563d27..3f3de6cc6 100644 --- a/pgcli-completion.bash +++ b/pgcli-completion.bash @@ -43,6 +43,10 @@ _pgcli() _pg_databases return 0 ;; + --env-file) + _filedir + return 0 + ;; --help|-v|--version|-p|--port|-R|--row-limit) # all other arguments are noop with these return 0 @@ -57,7 +61,7 @@ _pgcli() --*) # return list of available options COMPREPLY=( $( compgen -W '--host --port --user --password --no-password - --single-connection --version --dbname --pgclirc --dsn + --single-connection --version --dbname --env-file --pgclirc --dsn --row-limit --help' -- "$cur" ) ) [[ $COMPREPLY == *= ]] && compopt -o nospace return 0 diff --git a/pgcli/main.py b/pgcli/main.py index 972b3c670..d8a86a76c 100644 --- a/pgcli/main.py +++ b/pgcli/main.py @@ -27,6 +27,7 @@ from cli_helpers.utils import strip_ansi from .explain_output_formatter import ExplainOutputFormatter import click +from click.core import ParameterSource import sqlparse from sqlparse import tokens as sqlparse_tokens import tzlocal @@ -35,6 +36,7 @@ import setproctitle except ImportError: setproctitle = None +from dotenv import dotenv_values from prompt_toolkit.completion import DynamicCompleter, ThreadedCompleter from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode from prompt_toolkit.shortcuts import PromptSession, CompleteStyle @@ -82,7 +84,7 @@ from getpass import getuser -from psycopg import DatabaseError, OperationalError, InterfaceError, Notify +from psycopg import DatabaseError, OperationalError, InterfaceError, Notify, ProgrammingError from psycopg.conninfo import make_conninfo, conninfo_to_dict from psycopg.errors import Diagnostic @@ -1584,6 +1586,28 @@ def echo_via_pager(self, text, color=None): click.echo_via_pager(text, color) +def read_env_database(env_file): + """Read a database URL without exporting unrelated project settings.""" + try: + with open(env_file, encoding="utf-8-sig") as stream: + values = dotenv_values(stream=stream, interpolate=False) + except (OSError, UnicodeError) as exc: + raise click.BadParameter("Could not read the environment file.", param_hint="--env-file") from exc + + for name in ("DATABASE_URL", "DATABASE_URI", "DB_URL", "DB_URI"): + value = values.get(name) + if value and value.strip(): + # Do not echo the URL: it may contain credentials. + if not value.startswith(("postgres://", "postgresql://")): + raise click.BadParameter(f"{name} must be a PostgreSQL URL.", param_hint="--env-file") + try: + conninfo_to_dict(value) + except ProgrammingError as exc: + raise click.BadParameter(f"{name} contains an invalid PostgreSQL URL.", param_hint="--env-file") from exc + return value + raise click.BadParameter("No non-empty DATABASE_URL, DATABASE_URI, DB_URL or DB_URI found.", param_hint="--env-file") + + @click.command() # Default host is '' so psycopg can default to either localhost or unix socket @click.option( @@ -1640,6 +1664,11 @@ def echo_via_pager(self, text, color=None): ) @click.option("-v", "--version", is_flag=True, help="Version of pgcli.") @click.option("-d", "--dbname", "dbname_opt", help="database name to connect to.") +@click.option( + "--env-file", + type=click.Path(exists=True, dir_okay=False, readable=True), + help="Read a database URL from this local .env file. Explicit connection arguments take precedence.", +) @click.option( "--pgclirc", default=config_location() + "config", @@ -1758,7 +1787,9 @@ def echo_via_pager(self, text, color=None): ) @click.argument("dbname", default=lambda: None, envvar="PGDATABASE", nargs=1) @click.argument("username", default=lambda: None, envvar="PGUSER", nargs=1) +@click.pass_context def cli( + ctx, dbname, username_opt, host, @@ -1789,6 +1820,7 @@ def cli( force_destructive: bool, input_files: tuple, connect_timeout: int | None, + env_file, ): if version: print("Version:", __version__) @@ -1838,6 +1870,15 @@ def cli( ) sys.exit(1) + env_database = None + if env_file and not any( + ctx.get_parameter_source(name) == ParameterSource.COMMANDLINE + for name in ("dbname", "dbname_opt", "username", "username_opt", "host", "port", "dsn") + ): + env_database = read_env_database(env_file) + # An explicitly selected file takes precedence over environment defaults. + dsn = "" + pgcli = PGCli( prompt_passwd, never_prompt, @@ -1866,12 +1907,12 @@ def cli( if dbname_opt and dbname: # work as psql: when database is given as option and argument use the argument as user username = dbname - database = dbname_opt or dbname or "" + database = env_database or dbname_opt or dbname or "" user = username_opt or username service = None if database.startswith("service="): service = database[8:] - elif os.getenv("PGSERVICE") is not None: + elif env_database is None and os.getenv("PGSERVICE") is not None: service = os.getenv("PGSERVICE") # because option --ping, --list or -l are not supposed to have a db name. # A connection string is not a db name though: a URI or a key=value conninfo @@ -2022,7 +2063,7 @@ def echo_error(msg: str): pgcli.logger.debug( "Launch Params: \n\tdatabase: %r\tuser: %r\thost: %r\tport: %r", - database, + "" if env_database else database, user, host, port, diff --git a/pyproject.toml b/pyproject.toml index 71ee953ed..33f30e5c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ requires-python = ">=3.10" dependencies = [ "pgspecial>=2.0.0", # Click 8.1.8 through 8.3.0 have broken pager invocation for multi-argument PAGER values, which causes behave test failures. - "click >= 4.1, != 8.1.8, != 8.2.*, != 8.3.0, < 9", + "click >= 8.0, != 8.1.8, != 8.2.*, != 8.3.0, < 9", "Pygments>=2.0", # Pygments has to be Capitalcased. # We still need to use pt-2 unless pt-3 released on Fedora32 # see: https://github.com/dbcli/pgcli/pull/1197 @@ -33,6 +33,7 @@ dependencies = [ "psycopg-binary >= 3.0.14; sys_platform == 'win32'", "sqlparse >=0.3.0,<0.7", "configobj >= 5.0.6", + "python-dotenv >= 1.0", "cli_helpers[styles] >= 2.15.2", # setproctitle is used to mask the password when running `ps` in command line. # But this is not necessary in Windows since the password is never shown in the @@ -134,4 +135,4 @@ exclude = [ [tool.pytest.ini_options] minversion = "6.0" addopts = "--capture=sys --showlocals -rxs" -testpaths = ["tests"] \ No newline at end of file +testpaths = ["tests"] diff --git a/tests/test_env_file.py b/tests/test_env_file.py new file mode 100644 index 000000000..9aea9341c --- /dev/null +++ b/tests/test_env_file.py @@ -0,0 +1,205 @@ +import os +from contextlib import ExitStack +from unittest import mock + +import pytest +import click +from click.testing import CliRunner +from configobj import ConfigObj +from psycopg.conninfo import conninfo_to_dict + +from pgcli.main import PGCli, cli, read_env_database, setproctitle + + +URL = "postgresql://file_user:secret@file_host:6543/file_db?sslmode=require" +STOP = RuntimeError("connection selected") + + +@pytest.fixture +def invoke(tmp_path, monkeypatch): + # setproctitle locates argv using the original environment on Linux. + # Initialize it before removing connection variables from that environment. + if setproctitle: + setproctitle.getproctitle() + for name in tuple(os.environ): + if name.startswith("PG") or name in ("DSN", "DATABASE_URL", "DATABASE_URI", "DB_URL", "DB_URI"): + monkeypatch.delenv(name) + monkeypatch.chdir(tmp_path) + config = tmp_path / "pgclirc" + config.write_text("[main]\n[alias_dsn]\nlocal = postgres://alias_host/alias_db\n", encoding="utf-8") + + def run(args): + with ExitStack() as stack: + connections = { + name: stack.enter_context(mock.patch.object(PGCli, name, side_effect=STOP)) + for name in ("connect_uri", "connect_dsn", "connect_service", "connect") + } + result = CliRunner().invoke(cli, ["--pgclirc", str(config), *args]) + return result, connections + + return run + + +def assert_connection(result, connections, method, *args, **kwargs): + assert result.exception is STOP, result.output + connections[method].assert_called_once_with(*args, **kwargs) + assert sum(connection.call_count for connection in connections.values()) == 1 + + +@pytest.mark.parametrize("name", ["DATABASE_URL", "DATABASE_URI", "DB_URL", "DB_URI"]) +@pytest.mark.parametrize("filename", [".env", ".env.local"]) +def test_env_file_url(invoke, tmp_path, name, filename): + env_file = tmp_path / filename + env_file.write_text(f"# project settings\nexport {name}='{URL}'\n", encoding="utf-8-sig") + result, connections = invoke(["--env-file", str(env_file)]) + assert_connection(result, connections, "connect_uri", URL) + + +def test_env_file_variable_precedence(invoke, tmp_path): + env_file = tmp_path / ".env" + env_file.write_text( + "DB_URI=postgres://fourth/db\nDB_URL=postgres://third/db\nDATABASE_URI=postgres://second/db\nDATABASE_URL=postgres://first/db\n", + encoding="utf-8", + ) + result, connections = invoke(["--env-file", str(env_file)]) + assert_connection(result, connections, "connect_uri", "postgres://first/db") + + +def test_env_file_skips_empty_variables(invoke, tmp_path): + env_file = tmp_path / ".env" + env_file.write_text(f"DATABASE_URL\nDATABASE_URI=\nDB_URL=' '\nDB_URI={URL}\n", encoding="utf-8") + result, connections = invoke(["--env-file", str(env_file)]) + assert_connection(result, connections, "connect_uri", URL) + + +def test_env_file_overrides_environment_defaults(invoke, tmp_path, monkeypatch): + for name, value in { + "PGDATABASE": "environment_db", + "PGHOST": "environment_host", + "PGPORT": "9876", + "PGUSER": "environment_user", + "PGSERVICE": "environment_service", + "DSN": "environment_alias", + "DATABASE_URL": "postgres://environment/db", + }.items(): + monkeypatch.setenv(name, value) + env_file = tmp_path / ".env" + env_file.write_text(f"DB_URL={URL}\nPGPASSWORD=not_exported\nUNRELATED=not_exported\n", encoding="utf-8") + before = dict(os.environ) + result, connections = invoke(["--env-file", str(env_file)]) + assert_connection(result, connections, "connect_uri", URL) + for name in (*before, "PGPASSWORD", "UNRELATED"): + # PGCli itself sets PAGER/LESS during initialization. + if name not in ("PAGER", "LESS"): + assert os.environ.get(name) == before.get(name) + + +@pytest.mark.parametrize( + "args, method, expected, kwargs", + [ + (["explicit_db"], "connect", ("explicit_db", "", None, 5432), {}), + (["-d", "explicit_db"], "connect", ("explicit_db", "", None, 5432), {}), + (["explicit_db", "explicit_user"], "connect", ("explicit_db", "", "explicit_user", 5432), {}), + (["-h", "explicit_host"], "connect", ("", "explicit_host", None, 5432), {}), + (["-p", "7654"], "connect", ("", "", None, 7654), {}), + (["-U", "explicit_user"], "connect", ("", "", "explicit_user", 5432), {}), + (["-u", "explicit_user"], "connect", ("", "", "explicit_user", 5432), {}), + (["-D", "local"], "connect_uri", ("postgres://alias_host/alias_db",), {}), + (["postgres://explicit/db"], "connect_uri", ("postgres://explicit/db",), {}), + (["host=explicit dbname=db"], "connect_dsn", ("host=explicit dbname=db",), {"user": None}), + (["service=explicit"], "connect_service", ("explicit", None), {}), + ], +) +def test_explicit_connection_takes_precedence(invoke, tmp_path, args, method, expected, kwargs): + env_file = tmp_path / ".env" + env_file.write_text(f"DB_URL={URL}\n", encoding="utf-8") + result, connections = invoke(["--env-file", str(env_file), *args]) + assert_connection(result, connections, method, *expected, **kwargs) + + +def test_env_file_is_opt_in(invoke, tmp_path, monkeypatch): + (tmp_path / ".env").write_text(f"DB_URL={URL}\n", encoding="utf-8") + monkeypatch.setenv("DB_URL", URL) + result, connections = invoke([]) + assert_connection(result, connections, "connect", "", "", None, 5432) + + +@pytest.mark.parametrize("option", ["--ping", "--list"]) +def test_env_file_with_connection_checks(invoke, tmp_path, option): + env_file = tmp_path / ".env" + env_file.write_text(f"DB_URL={URL}\n", encoding="utf-8") + result, connections = invoke(["--env-file", str(env_file), option]) + assert_connection(result, connections, "connect_uri", URL) + + +def test_env_file_list_without_database(invoke, tmp_path): + env_file = tmp_path / ".env" + env_file.write_text("DB_URL=postgres://user@host:6543?sslmode=require\n", encoding="utf-8") + result, connections = invoke(["--env-file", str(env_file), "--list"]) + assert result.exception is STOP, result.output + assert conninfo_to_dict(connections["connect_dsn"].call_args.args[0]) == { + "user": "user", + "host": "host", + "port": "6543", + "dbname": "postgres", + "sslmode": "require", + } + + +@pytest.mark.parametrize( + "contents, message", + [ + ("", "No non-empty"), + ("UNRELATED=value\n", "No non-empty"), + ("DB_URL=\n", "No non-empty"), + ("DB_URL=mysql://user:secret@host/db\n", "must be a PostgreSQL URL"), + ("DB_URL=postgres://user:secret@host/db?unknown_parameter=secret\n", "invalid PostgreSQL URL"), + (b"DB_URL=\xff", "Could not read"), + ], +) +def test_env_file_errors(invoke, tmp_path, contents, message): + env_file = tmp_path / ".env" + env_file.write_bytes(contents if isinstance(contents, bytes) else contents.encode("utf-8")) + result, connections = invoke(["--env-file", str(env_file)]) + assert result.exit_code == 2 + assert message in result.output + assert "secret" not in result.output + assert not any(connection.called for connection in connections.values()) + + +@pytest.mark.parametrize("filename", ["missing", "."]) +def test_env_file_path_errors(invoke, filename): + result, connections = invoke(["--env-file", filename]) + assert result.exit_code == 2 + assert "--env-file" in result.output + assert not any(connection.called for connection in connections.values()) + + +def test_env_file_read_error(): + with mock.patch("pgcli.main.open", side_effect=PermissionError), pytest.raises(click.BadParameter, match="Could not read"): + read_env_database(".env") + + +def test_env_file_does_not_interpolate(invoke, tmp_path, monkeypatch): + monkeypatch.setenv("PASSWORD", "expanded") + url = "postgres://user:${PASSWORD}@host/db" + env_file = tmp_path / ".env" + env_file.write_text(f"DB_URL='{url}'\n", encoding="utf-8") + result, connections = invoke(["--env-file", str(env_file)]) + assert_connection(result, connections, "connect_uri", url) + + +def test_env_file_url_is_not_logged(invoke, tmp_path, monkeypatch): + env_file = tmp_path / ".env" + env_file.write_text(f"DB_URL={URL}\n", encoding="utf-8") + monkeypatch.setattr("pgcli.main.setproctitle", None) + with mock.patch("pgcli.main.PGCli") as pgcli: + instance = pgcli.return_value + instance.config = ConfigObj({"main": {"use_local_timezone": "False"}}) + result = CliRunner().invoke(cli, ["--env-file", str(env_file)]) + assert result.exit_code == 0, result.output + instance.connect_uri.assert_called_once_with(URL) + instance.run_cli.assert_called_once() + instance.logger.debug.assert_called_once() + assert URL not in str(instance.logger.debug.call_args) + assert "" in str(instance.logger.debug.call_args)